2024-09-18T00:47:21.192 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-18T00:47:21.214 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-18T00:47:21.311 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686 branch: squid description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-lz4 policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock} email: ceph-qa@ceph.com first_in_suite: false job_id: '7909686' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run four rbd-mirror daemons per cluster - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon name: teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: simple client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluestore block size: 96636764160 bluestore compression algorithm: lz4 bluestore compression mode: aggressive bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f selinux: allowlist: - scontext=system_u:system_r:getty_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: squid sha1: b0d356b98e08f6bc59ef15652f0d0e4298fc83c7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 1983 sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f sleep_before_teardown: 0 subset: 79/128 suite: rbd suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_b0d356b98e08f6bc59ef15652f0d0e4298fc83c7/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: b0d356b98e08f6bc59ef15652f0d0e4298fc83c7 targets: smithi078.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHoBQr+C0uE2Le/lt0B2M0DhCOE+f9NQs6lJvI49ArMnn1SGLqMsYEr4xHrV0h9ypsUL6i4u9Z2dZQH3T3Eifkc= smithi150.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAkxbPf3Tcg64mNNL1gjy0wPGGOSlVPApAHiVtGuSX3ZCnVTGXln5Ad+K5u/FeK/vmT65DuVNC0Z+brnbLwhT6k= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - rbd-mirror: client: cluster1.client.mirror.0 thrash: true - rbd-mirror: client: cluster1.client.mirror.1 thrash: true - rbd-mirror: client: cluster1.client.mirror.2 thrash: true - rbd-mirror: client: cluster1.client.mirror.3 thrash: true - rbd-mirror: client: cluster2.client.mirror.0 thrash: true - rbd-mirror: client: cluster2.client.mirror.1 thrash: true - rbd-mirror: client: cluster2.client.mirror.2 thrash: true - rbd-mirror: client: cluster2.client.mirror.3 thrash: true - rbd-mirror-thrash: cluster: cluster1 - rbd-mirror-thrash: cluster: cluster2 - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_stress.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: image RBD_IMAGE_FEATURES: layering,exclusive-lock RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: snapshot RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-17_21:16:02 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-18T00:47:21.311 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_b0d356b98e08f6bc59ef15652f0d0e4298fc83c7/qa; will attempt to use it 2024-09-18T00:47:21.312 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_b0d356b98e08f6bc59ef15652f0d0e4298fc83c7/qa/tasks 2024-09-18T00:47:21.312 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-18T00:47:21.314 INFO:teuthology.task.internal:Checking packages... 2024-09-18T00:47:21.335 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'c2ddf9721ef254645201cdbabd9c8f49c30a586f' 2024-09-18T00:47:21.336 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-18T00:47:21.336 INFO:teuthology.packaging:ref: None 2024-09-18T00:47:21.336 INFO:teuthology.packaging:tag: None 2024-09-18T00:47:21.336 INFO:teuthology.packaging:branch: squid 2024-09-18T00:47:21.336 INFO:teuthology.packaging:sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-18T00:47:21.336 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-18T00:47:21.565 INFO:teuthology.task.internal:Found packages for ceph version 19.2.0-0 2024-09-18T00:47:21.566 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-18T00:47:21.573 INFO:teuthology.task.internal:no buildpackages task found 2024-09-18T00:47:21.573 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-18T00:47:21.621 INFO:teuthology.task.internal:Saving configuration 2024-09-18T00:47:21.638 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-18T00:47:21.646 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-18T00:47:21.669 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi078.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-18 00:42:10.921626', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHoBQr+C0uE2Le/lt0B2M0DhCOE+f9NQs6lJvI49ArMnn1SGLqMsYEr4xHrV0h9ypsUL6i4u9Z2dZQH3T3Eifkc='} 2024-09-18T00:47:21.685 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi150.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-18 00:42:10.919677', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAkxbPf3Tcg64mNNL1gjy0wPGGOSlVPApAHiVtGuSX3ZCnVTGXln5Ad+K5u/FeK/vmT65DuVNC0Z+brnbLwhT6k='} 2024-09-18T00:47:21.685 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-18T00:47:21.692 INFO:teuthology.task.internal:roles: ubuntu@smithi078.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2024-09-18T00:47:21.692 INFO:teuthology.task.internal:roles: ubuntu@smithi150.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2024-09-18T00:47:21.692 INFO:teuthology.run_tasks:Running task console_log... 2024-09-18T00:47:21.756 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f096a847ac0>, signals=[15]) 2024-09-18T00:47:21.756 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-18T00:47:21.762 INFO:teuthology.task.internal:Opening connections... 2024-09-18T00:47:21.762 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi078.front.sepia.ceph.com 2024-09-18T00:47:21.763 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T00:47:21.840 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi150.front.sepia.ceph.com 2024-09-18T00:47:21.841 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T00:47:21.913 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-18T00:47:21.919 DEBUG:teuthology.orchestra.run.smithi078:> uname -m 2024-09-18T00:47:21.938 INFO:teuthology.orchestra.run.smithi078.stdout:x86_64 2024-09-18T00:47:21.938 DEBUG:teuthology.orchestra.run.smithi078:> cat /etc/os-release 2024-09-18T00:47:21.996 INFO:teuthology.orchestra.run.smithi078.stdout:NAME="CentOS Stream" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION="9" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:ID="centos" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:ID_LIKE="rhel fedora" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION_ID="9" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:PLATFORM_ID="platform:el9" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:ANSI_COLOR="0;31" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:LOGO="fedora-logo-icon" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:HOME_URL="https://centos.org/" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-18T00:47:21.997 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-18T00:47:21.998 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-18T00:47:21.998 INFO:teuthology.lock.ops:Updating smithi078.front.sepia.ceph.com on lock server 2024-09-18T00:47:22.023 DEBUG:teuthology.orchestra.run.smithi150:> uname -m 2024-09-18T00:47:22.042 INFO:teuthology.orchestra.run.smithi150.stdout:x86_64 2024-09-18T00:47:22.042 DEBUG:teuthology.orchestra.run.smithi150:> cat /etc/os-release 2024-09-18T00:47:22.100 INFO:teuthology.orchestra.run.smithi150.stdout:NAME="CentOS Stream" 2024-09-18T00:47:22.100 INFO:teuthology.orchestra.run.smithi150.stdout:VERSION="9" 2024-09-18T00:47:22.100 INFO:teuthology.orchestra.run.smithi150.stdout:ID="centos" 2024-09-18T00:47:22.100 INFO:teuthology.orchestra.run.smithi150.stdout:ID_LIKE="rhel fedora" 2024-09-18T00:47:22.100 INFO:teuthology.orchestra.run.smithi150.stdout:VERSION_ID="9" 2024-09-18T00:47:22.100 INFO:teuthology.orchestra.run.smithi150.stdout:PLATFORM_ID="platform:el9" 2024-09-18T00:47:22.100 INFO:teuthology.orchestra.run.smithi150.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-18T00:47:22.100 INFO:teuthology.orchestra.run.smithi150.stdout:ANSI_COLOR="0;31" 2024-09-18T00:47:22.101 INFO:teuthology.orchestra.run.smithi150.stdout:LOGO="fedora-logo-icon" 2024-09-18T00:47:22.101 INFO:teuthology.orchestra.run.smithi150.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-18T00:47:22.101 INFO:teuthology.orchestra.run.smithi150.stdout:HOME_URL="https://centos.org/" 2024-09-18T00:47:22.101 INFO:teuthology.orchestra.run.smithi150.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-18T00:47:22.101 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-18T00:47:22.101 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-18T00:47:22.101 INFO:teuthology.lock.ops:Updating smithi150.front.sepia.ceph.com on lock server 2024-09-18T00:47:22.124 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-18T00:47:22.133 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-18T00:47:22.214 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-18T00:47:22.214 DEBUG:teuthology.orchestra.run.smithi078:> test '!' -e /home/ubuntu/cephtest 2024-09-18T00:47:22.217 DEBUG:teuthology.orchestra.run.smithi150:> test '!' -e /home/ubuntu/cephtest 2024-09-18T00:47:22.235 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-18T00:47:22.241 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-18T00:47:22.241 DEBUG:teuthology.orchestra.run.smithi078:> test -z $(ls -A /var/lib/ceph) 2024-09-18T00:47:22.275 DEBUG:teuthology.orchestra.run.smithi150:> test -z $(ls -A /var/lib/ceph) 2024-09-18T00:47:22.307 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-18T00:47:22.362 INFO:teuthology.run_tasks:Running task kernel... 2024-09-18T00:47:22.376 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-18T00:47:22.376 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'kdb': 1, 'sha1': 'distro'}, 'cluster2.mon.a': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-18T00:47:22.376 DEBUG:teuthology.orchestra.run.smithi078:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-18T00:47:22.377 DEBUG:teuthology.orchestra.run.smithi150:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-18T00:47:22.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T00:47:22.394 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-09-18T00:47:22.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T00:47:22.395 DEBUG:teuthology.orchestra.run.smithi150:> uname -r 2024-09-18T00:47:22.452 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-503.el9.x86_64 2024-09-18T00:47:22.452 INFO:teuthology.task.kernel:Running kernel on smithi078: 5.14.0-503.el9.x86_64 2024-09-18T00:47:22.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum install -y kernel 2024-09-18T00:47:22.453 INFO:teuthology.orchestra.run.smithi150.stdout:5.14.0-503.el9.x86_64 2024-09-18T00:47:22.453 INFO:teuthology.task.kernel:Running kernel on smithi150: 5.14.0-503.el9.x86_64 2024-09-18T00:47:22.454 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum install -y kernel 2024-09-18T00:47:25.967 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - BaseOS 4.6 MB/s | 8.3 MB 00:01 2024-09-18T00:47:26.009 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - BaseOS 4.5 MB/s | 8.3 MB 00:01 2024-09-18T00:47:30.851 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - AppStream 5.4 MB/s | 20 MB 00:03 2024-09-18T00:47:34.043 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - AppStream 2.9 MB/s | 20 MB 00:07 2024-09-18T00:47:37.935 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - CRB 2.0 MB/s | 6.5 MB 00:03 2024-09-18T00:47:39.693 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - Extras packages 34 kB/s | 19 kB 00:00 2024-09-18T00:47:39.893 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - CRB 3.6 MB/s | 6.5 MB 00:01 2024-09-18T00:47:41.298 INFO:teuthology.orchestra.run.smithi078.stdout:Extra Packages for Enterprise Linux 15 MB/s | 23 MB 00:01 2024-09-18T00:47:41.719 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - Extras packages 42 kB/s | 19 kB 00:00 2024-09-18T00:47:43.423 INFO:teuthology.orchestra.run.smithi150.stdout:Extra Packages for Enterprise Linux 14 MB/s | 23 MB 00:01 2024-09-18T00:47:46.621 INFO:teuthology.orchestra.run.smithi078.stdout:lab-extras 36 kB/s | 1.7 kB 00:00 2024-09-18T00:47:48.461 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-18T00:47:48.461 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-18T00:47:48.540 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:47:48.551 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:47:48.552 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-09-18T00:47:48.552 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:47:48.552 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:47:48.552 INFO:teuthology.orchestra.run.smithi078.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-18T00:47:48.553 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-18T00:47:48.553 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-18T00:47:48.553 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-18T00:47:48.553 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-18T00:47:48.553 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:47:48.554 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:47:48.554 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:47:48.554 INFO:teuthology.orchestra.run.smithi078.stdout:Install 4 Packages 2024-09-18T00:47:48.554 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:47:48.554 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 79 M 2024-09-18T00:47:48.554 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 126 M 2024-09-18T00:47:48.555 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:47:48.755 INFO:teuthology.orchestra.run.smithi078.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 344 kB/s | 36 kB 00:00 2024-09-18T00:47:48.837 INFO:teuthology.orchestra.run.smithi150.stdout:lab-extras 21 kB/s | 1.7 kB 00:00 2024-09-18T00:47:49.131 INFO:teuthology.orchestra.run.smithi078.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-09-18T00:47:49.565 INFO:teuthology.orchestra.run.smithi078.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 38 MB/s | 35 MB 00:00 2024-09-18T00:47:49.840 INFO:teuthology.orchestra.run.smithi078.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 26 MB/s | 29 MB 00:01 2024-09-18T00:47:49.840 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:47:49.840 INFO:teuthology.orchestra.run.smithi078.stdout:Total 61 MB/s | 79 MB 00:01 2024-09-18T00:47:50.521 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:47:50.735 INFO:teuthology.orchestra.run.smithi150.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-18T00:47:50.736 INFO:teuthology.orchestra.run.smithi150.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-18T00:47:50.826 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:47:50.833 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:47:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: Package Architecture Version Repository Size 2024-09-18T00:47:50.834 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:47:50.834 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:47:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-18T00:47:50.834 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-09-18T00:47:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-18T00:47:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-18T00:47:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-18T00:47:50.834 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:47:50.834 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:47:50.834 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:47:50.835 INFO:teuthology.orchestra.run.smithi150.stdout:Install 4 Packages 2024-09-18T00:47:50.835 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:47:50.835 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 79 M 2024-09-18T00:47:50.835 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 126 M 2024-09-18T00:47:50.835 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:47:50.882 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:47:50.882 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:47:51.030 INFO:teuthology.orchestra.run.smithi150.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 462 kB/s | 36 kB 00:00 2024-09-18T00:47:51.489 INFO:teuthology.orchestra.run.smithi150.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 29 MB/s | 16 MB 00:00 2024-09-18T00:47:51.870 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:47:51.870 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:47:52.072 INFO:teuthology.orchestra.run.smithi150.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 31 MB/s | 35 MB 00:01 2024-09-18T00:47:52.532 INFO:teuthology.orchestra.run.smithi150.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 19 MB/s | 29 MB 00:01 2024-09-18T00:47:52.532 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:47:52.532 INFO:teuthology.orchestra.run.smithi150.stdout:Total 46 MB/s | 79 MB 00:01 2024-09-18T00:47:53.211 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:47:53.609 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:47:53.610 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:47:54.292 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:47:54.582 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:47:54.582 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:47:54.960 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-18T00:47:55.041 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T00:47:57.018 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:47:57.667 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-18T00:47:57.757 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T00:47:57.795 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T00:47:57.928 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T00:48:00.531 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T00:48:00.683 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T00:48:02.766 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T00:48:02.853 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:05.516 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T00:48:05.625 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:07.704 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:10.486 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:45.568 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:45.607 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:45.981 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:45.981 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-18T00:48:45.982 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T00:48:45.982 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T00:48:46.813 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:46.813 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:48:46.813 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:48:46.813 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-18T00:48:46.813 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-18T00:48:46.813 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-18T00:48:46.813 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-18T00:48:46.814 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:48:46.814 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:48:47.185 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-09-18T00:48:47.263 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-18T00:48:47.263 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-18T00:48:47.263 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-18T00:48:47.264 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-18T00:48:47.264 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-18T00:48:47.264 INFO:teuthology.task.kernel:Installing distro kernel on cluster1.mon.a... 2024-09-18T00:48:47.264 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi078.front.sepia.ceph.com, path=None, version=distro) 2024-09-18T00:48:47.264 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-09-18T00:48:47.323 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-18T00:48:47.323 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-18T00:48:47.323 DEBUG:teuthology.orchestra.run.smithi078:> sudo rpm -qi grub2-tools 2024-09-18T00:48:47.365 INFO:teuthology.orchestra.run.smithi078.stdout:Name : grub2-tools 2024-09-18T00:48:47.365 INFO:teuthology.orchestra.run.smithi078.stdout:Epoch : 1 2024-09-18T00:48:47.366 INFO:teuthology.orchestra.run.smithi078.stdout:Version : 2.06 2024-09-18T00:48:47.366 INFO:teuthology.orchestra.run.smithi078.stdout:Release : 82.el9 2024-09-18T00:48:47.366 INFO:teuthology.orchestra.run.smithi078.stdout:Architecture: x86_64 2024-09-18T00:48:47.366 INFO:teuthology.orchestra.run.smithi078.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-18T00:48:47.366 INFO:teuthology.orchestra.run.smithi078.stdout:Group : Unspecified 2024-09-18T00:48:47.366 INFO:teuthology.orchestra.run.smithi078.stdout:Size : 8274670 2024-09-18T00:48:47.366 INFO:teuthology.orchestra.run.smithi078.stdout:License : GPLv3+ 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:Packager : builder@centos.org 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:Vendor : CentOS 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:Summary : Support tools for GRUB. 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:Description : 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-18T00:48:47.367 INFO:teuthology.orchestra.run.smithi078.stdout:hardware devices. 2024-09-18T00:48:47.368 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:48:47.368 INFO:teuthology.orchestra.run.smithi078.stdout:This subpackage provides tools for support of all platforms. 2024-09-18T00:48:47.368 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-18T00:48:47.368 INFO:teuthology.task.kernel:Updating grub on smithi078 to boot 5.14.0-508.el9.x86_64 2024-09-18T00:48:47.369 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-18T00:48:48.005 INFO:teuthology.orchestra.run.smithi078.stderr:Generating grub configuration file ... 2024-09-18T00:48:48.546 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:48.586 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:48.976 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:48.977 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-18T00:48:48.977 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-18T00:48:48.977 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-18T00:48:49.292 INFO:teuthology.orchestra.run.smithi078.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-18T00:48:49.312 INFO:teuthology.orchestra.run.smithi078.stderr:done 2024-09-18T00:48:49.317 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2024-09-18T00:48:49.333 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.8e0RJ6po92 2024-09-18T00:48:49.333 DEBUG:teuthology.orchestra.run.smithi078:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.8e0RJ6po92 2024-09-18T00:48:49.408 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.8e0RJ6po92 2024-09-18T00:48:49.554 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.8e0RJ6po92 is 6KB 2024-09-18T00:48:49.602 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.8e0RJ6po92 2024-09-18T00:48:49.616 DEBUG:teuthology.orchestra.run.smithi078:> sudo /bin/ls /boot/loader/entries || true 2024-09-18T00:48:49.688 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-18T00:48:49.688 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-18T00:48:49.688 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-18T00:48:49.689 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-18T00:48:49.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-18T00:48:49.820 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-18T00:48:49.820 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:48:49.821 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:48:49.821 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-18T00:48:49.821 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-18T00:48:49.821 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-18T00:48:49.821 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-18T00:48:49.821 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:48:49.821 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:48:49.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo shutdown -r now 2024-09-18T00:48:50.157 DEBUG:teuthology.orchestra.run.smithi150:> rpm -q kernel | sort -rV | head -n 1 2024-09-18T00:48:50.287 INFO:teuthology.orchestra.run.smithi150.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-18T00:48:50.287 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-18T00:48:50.287 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-18T00:48:50.287 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-18T00:48:50.287 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-18T00:48:50.287 INFO:teuthology.task.kernel:Installing distro kernel on cluster2.mon.a... 2024-09-18T00:48:50.288 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi150.front.sepia.ceph.com, path=None, version=distro) 2024-09-18T00:48:50.288 DEBUG:teuthology.orchestra.run.smithi150:> rpm -q kernel | sort -rV | head -n 1 2024-09-18T00:48:50.346 INFO:teuthology.orchestra.run.smithi150.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-18T00:48:50.346 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-18T00:48:50.346 DEBUG:teuthology.orchestra.run.smithi150:> sudo rpm -qi grub2-tools 2024-09-18T00:48:50.378 INFO:teuthology.orchestra.run.smithi150.stdout:Name : grub2-tools 2024-09-18T00:48:50.378 INFO:teuthology.orchestra.run.smithi150.stdout:Epoch : 1 2024-09-18T00:48:50.378 INFO:teuthology.orchestra.run.smithi150.stdout:Version : 2.06 2024-09-18T00:48:50.378 INFO:teuthology.orchestra.run.smithi150.stdout:Release : 82.el9 2024-09-18T00:48:50.378 INFO:teuthology.orchestra.run.smithi150.stdout:Architecture: x86_64 2024-09-18T00:48:50.378 INFO:teuthology.orchestra.run.smithi150.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:Group : Unspecified 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:Size : 8274670 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:License : GPLv3+ 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:Packager : builder@centos.org 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:Vendor : CentOS 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:Summary : Support tools for GRUB. 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:Description : 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:48:50.379 INFO:teuthology.orchestra.run.smithi150.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-18T00:48:50.380 INFO:teuthology.orchestra.run.smithi150.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-18T00:48:50.380 INFO:teuthology.orchestra.run.smithi150.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-18T00:48:50.380 INFO:teuthology.orchestra.run.smithi150.stdout:hardware devices. 2024-09-18T00:48:50.380 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:48:50.380 INFO:teuthology.orchestra.run.smithi150.stdout:This subpackage provides tools for support of all platforms. 2024-09-18T00:48:50.381 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-18T00:48:50.381 INFO:teuthology.task.kernel:Updating grub on smithi150 to boot 5.14.0-508.el9.x86_64 2024-09-18T00:48:50.381 DEBUG:teuthology.orchestra.run.smithi150:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-18T00:48:51.035 INFO:teuthology.orchestra.run.smithi150.stderr:Generating grub configuration file ... 2024-09-18T00:48:52.416 INFO:teuthology.orchestra.run.smithi150.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-18T00:48:52.443 INFO:teuthology.orchestra.run.smithi150.stderr:done 2024-09-18T00:48:52.445 DEBUG:teuthology.orchestra.run.smithi150:> mktemp 2024-09-18T00:48:52.461 INFO:teuthology.orchestra.run.smithi150.stdout:/tmp/tmp.ja0ScG4NEW 2024-09-18T00:48:52.461 DEBUG:teuthology.orchestra.run.smithi150:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.ja0ScG4NEW 2024-09-18T00:48:52.528 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 0666 /tmp/tmp.ja0ScG4NEW 2024-09-18T00:48:52.665 DEBUG:teuthology.orchestra.remote:smithi150:/tmp/tmp.ja0ScG4NEW is 6KB 2024-09-18T00:48:52.687 DEBUG:teuthology.orchestra.run.smithi150:> rm -fr /tmp/tmp.ja0ScG4NEW 2024-09-18T00:48:52.703 DEBUG:teuthology.orchestra.run.smithi150:> sudo /bin/ls /boot/loader/entries || true 2024-09-18T00:48:52.769 INFO:teuthology.orchestra.run.smithi150.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-18T00:48:52.769 INFO:teuthology.orchestra.run.smithi150.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-18T00:48:52.769 INFO:teuthology.orchestra.run.smithi150.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-18T00:48:52.769 INFO:teuthology.orchestra.run.smithi150.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-18T00:48:52.770 DEBUG:teuthology.orchestra.run.smithi150:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-18T00:48:52.929 DEBUG:teuthology.orchestra.run.smithi150:> sudo shutdown -r now 2024-09-18T00:49:19.882 INFO:teuthology.task.kernel:Checking client cluster1.mon.a for new kernel version... 2024-09-18T00:49:19.882 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-18T00:49:19.883 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T00:49:22.933 INFO:teuthology.task.kernel:Checking client cluster2.mon.a for new kernel version... 2024-09-18T00:49:22.933 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-09-18T00:49:22.934 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T00:49:38.365 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-18T00:49:47.373 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-18T00:49:47.374 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T00:49:50.429 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-18T00:49:57.565 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.150 2024-09-18T00:50:02.434 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-18T00:50:02.435 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T00:50:06.569 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-09-18T00:50:06.569 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T00:50:24.957 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.150 2024-09-18T00:50:34.331 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-09-18T00:50:34.719 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-18T00:50:34.719 INFO:teuthology.task.kernel:Checking kernel version of cluster1.mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-18T00:50:34.719 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-09-18T00:50:34.733 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-508.el9.x86_64 2024-09-18T00:50:34.733 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-18T00:50:34.734 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-18T00:50:34.734 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-18T00:50:35.735 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2024-09-18T00:50:35.735 DEBUG:teuthology.orchestra.run.smithi078:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-18T00:50:35.865 INFO:teuthology.orchestra.run.smithi078.stdout:ttyS1 2024-09-18T00:50:35.887 DEBUG:teuthology.parallel:result is None 2024-09-18T00:50:36.959 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-09-18T00:50:36.960 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T00:50:37.280 DEBUG:teuthology.orchestra.run.smithi150:> true 2024-09-18T00:50:37.737 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-09-18T00:50:37.737 INFO:teuthology.task.kernel:Checking kernel version of cluster2.mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-18T00:50:37.737 DEBUG:teuthology.orchestra.run.smithi150:> uname -r 2024-09-18T00:50:37.765 INFO:teuthology.orchestra.run.smithi150.stdout:5.14.0-508.el9.x86_64 2024-09-18T00:50:37.765 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-18T00:50:37.765 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-18T00:50:37.765 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-18T00:50:38.766 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2024-09-18T00:50:38.766 DEBUG:teuthology.orchestra.run.smithi150:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-18T00:50:38.878 INFO:teuthology.orchestra.run.smithi150.stdout:ttyS1 2024-09-18T00:50:38.903 DEBUG:teuthology.parallel:result is None 2024-09-18T00:50:38.903 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-18T00:50:38.913 INFO:teuthology.task.internal:Creating test directory... 2024-09-18T00:50:38.914 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-18T00:50:38.916 DEBUG:teuthology.orchestra.run.smithi150:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-18T00:50:38.932 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-18T00:50:38.943 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-18T00:50:38.980 INFO:teuthology.task.internal:Creating archive directory... 2024-09-18T00:50:38.980 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-18T00:50:38.983 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-18T00:50:39.025 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-18T00:50:39.036 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-18T00:50:39.036 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-18T00:50:39.061 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-18T00:50:39.097 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-18T00:50:39.107 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-18T00:50:39.108 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-18T00:50:39.121 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-18T00:50:39.121 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-18T00:50:39.129 INFO:teuthology.task.internal:Configuring sudo... 2024-09-18T00:50:39.130 DEBUG:teuthology.orchestra.run.smithi078:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-18T00:50:39.132 DEBUG:teuthology.orchestra.run.smithi150:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-18T00:50:39.175 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-18T00:50:39.187 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-18T00:50:39.188 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-18T00:50:39.199 DEBUG:teuthology.orchestra.run.smithi150:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-18T00:50:39.230 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-18T00:50:39.291 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-18T00:50:39.359 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-18T00:50:39.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-18T00:50:39.441 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:50:39.441 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-18T00:50:39.506 DEBUG:teuthology.orchestra.run.smithi150:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-18T00:50:39.547 DEBUG:teuthology.orchestra.run.smithi150:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-18T00:50:39.622 DEBUG:teuthology.orchestra.run.smithi150:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-18T00:50:39.661 DEBUG:teuthology.orchestra.run.smithi150:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-18T00:50:39.705 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:50:39.705 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-18T00:50:39.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo service rsyslog restart 2024-09-18T00:50:39.772 DEBUG:teuthology.orchestra.run.smithi150:> sudo service rsyslog restart 2024-09-18T00:50:39.821 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-18T00:50:39.873 INFO:teuthology.orchestra.run.smithi150.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-18T00:50:40.202 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-18T00:50:40.210 INFO:teuthology.task.internal:Starting timer... 2024-09-18T00:50:40.210 INFO:teuthology.run_tasks:Running task pcp... 2024-09-18T00:50:40.249 INFO:teuthology.run_tasks:Running task selinux... 2024-09-18T00:50:40.281 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:getty_t:s0']} 2024-09-18T00:50:40.281 DEBUG:teuthology.orchestra.run.smithi078:> sudo service auditd rotate 2024-09-18T00:50:40.328 INFO:teuthology.orchestra.run.smithi078.stdout:Rotating logs: 2024-09-18T00:50:40.330 DEBUG:teuthology.orchestra.run.smithi150:> sudo service auditd rotate 2024-09-18T00:50:40.377 INFO:teuthology.orchestra.run.smithi150.stdout:Rotating logs: 2024-09-18T00:50:40.379 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-18T00:50:40.380 DEBUG:teuthology.orchestra.run.smithi078:> /usr/sbin/getenforce 2024-09-18T00:50:40.423 INFO:teuthology.orchestra.run.smithi078.stdout:Permissive 2024-09-18T00:50:40.424 DEBUG:teuthology.orchestra.run.smithi150:> /usr/sbin/getenforce 2024-09-18T00:50:40.456 INFO:teuthology.orchestra.run.smithi150.stdout:Permissive 2024-09-18T00:50:40.456 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi078.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi150.front.sepia.ceph.com': 'permissive'} 2024-09-18T00:50:40.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-18T00:50:40.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T00:50:40.490 DEBUG:teuthology.orchestra.run.smithi150:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-18T00:50:40.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T00:50:40.522 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-18T00:50:40.522 DEBUG:teuthology.orchestra.run.smithi078:> sudo /usr/sbin/setenforce permissive 2024-09-18T00:50:40.562 DEBUG:teuthology.orchestra.run.smithi150:> sudo /usr/sbin/setenforce permissive 2024-09-18T00:50:40.594 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-18T00:50:40.604 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-18T00:50:40.616 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-18T00:50:40.617 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi078.front.sepia.ceph.com,smithi150.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-18T00:54:24.770 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi078.front.sepia.ceph.com'), Remote(name='ubuntu@smithi150.front.sepia.ceph.com')] 2024-09-18T00:54:24.772 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-18T00:54:24.773 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T00:54:24.852 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-09-18T00:54:24.928 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-18T00:54:24.928 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-09-18T00:54:24.929 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-18T00:54:25.002 DEBUG:teuthology.orchestra.run.smithi150:> true 2024-09-18T00:54:25.080 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-09-18T00:54:25.080 INFO:teuthology.run_tasks:Running task clock... 2024-09-18T00:54:25.091 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-18T00:54:25.091 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-18T00:54:25.092 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-18T00:54:25.095 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-18T00:54:25.095 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-18T00:54:25.120 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-18T00:54:25.134 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-18T00:54:25.150 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-18T00:54:25.158 INFO:teuthology.orchestra.run.smithi078.stderr:sudo: ntpd: command not found 2024-09-18T00:54:25.164 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-18T00:54:25.169 INFO:teuthology.orchestra.run.smithi078.stdout:506 Cannot talk to daemon 2024-09-18T00:54:25.184 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-18T00:54:25.188 INFO:teuthology.orchestra.run.smithi150.stderr:sudo: ntpd: command not found 2024-09-18T00:54:25.197 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-18T00:54:25.199 INFO:teuthology.orchestra.run.smithi150.stdout:506 Cannot talk to daemon 2024-09-18T00:54:25.213 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-18T00:54:25.227 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-18T00:54:25.251 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-09-18T00:54:25.255 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-18T00:54:25.255 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-09-18T00:54:25.255 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T00:54:25.255 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T00:54:25.256 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T00:54:25.256 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T00:54:25.279 INFO:teuthology.orchestra.run.smithi150.stderr:bash: line 1: ntpq: command not found 2024-09-18T00:54:25.283 INFO:teuthology.orchestra.run.smithi150.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-18T00:54:25.283 INFO:teuthology.orchestra.run.smithi150.stdout:=============================================================================== 2024-09-18T00:54:25.283 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T00:54:25.283 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T00:54:25.283 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T00:54:25.283 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T00:54:25.284 INFO:teuthology.run_tasks:Running task install... 2024-09-18T00:54:25.293 DEBUG:teuthology.task.install:project ceph 2024-09-18T00:54:25.293 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}} 2024-09-18T00:54:25.293 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'} 2024-09-18T00:54:25.293 INFO:teuthology.task.install:Using flavor: default 2024-09-18T00:54:25.300 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-18T00:54:25.300 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2024-09-18T00:54:25.300 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f', 'tag': None, 'wait_for_package': False} 2024-09-18T00:54:25.301 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-18T00:54:25.304 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f', 'tag': None, 'wait_for_package': False} 2024-09-18T00:54:25.304 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-18T00:54:25.445 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/centos/9/flavors/default/ 2024-09-18T00:54:25.445 INFO:teuthology.task.install.rpm:Package version is 19.1.1-330.gc2ddf972 2024-09-18T00:54:25.451 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/centos/9/flavors/default/ 2024-09-18T00:54:25.451 INFO:teuthology.task.install.rpm:Package version is 19.1.1-330.gc2ddf972 2024-09-18T00:54:25.555 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-18T00:54:25.556 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:54:25.556 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-18T00:54:25.562 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/squid/c2ddf9721ef254645201cdbabd9c8f49c30a586f/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-18T00:54:25.562 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:54:25.562 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-18T00:54:25.581 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2024-09-18T00:54:25.582 DEBUG:teuthology.orchestra.run.smithi078:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/c2ddf9721ef254645201cdbabd9c8f49c30a586f/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-18T00:54:25.587 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2024-09-18T00:54:25.587 DEBUG:teuthology.orchestra.run.smithi150:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/c2ddf9721ef254645201cdbabd9c8f49c30a586f/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-18T00:54:25.647 DEBUG:teuthology.orchestra.run.smithi078:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-18T00:54:25.653 DEBUG:teuthology.orchestra.run.smithi150:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-18T00:54:25.721 DEBUG:teuthology.orchestra.run.smithi078:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-18T00:54:25.728 DEBUG:teuthology.orchestra.run.smithi150:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-18T00:54:25.785 INFO:teuthology.orchestra.run.smithi078.stdout:check_obsoletes = 1 2024-09-18T00:54:25.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum clean all 2024-09-18T00:54:25.793 INFO:teuthology.orchestra.run.smithi150.stdout:check_obsoletes = 1 2024-09-18T00:54:25.794 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum clean all 2024-09-18T00:54:26.020 INFO:teuthology.orchestra.run.smithi150.stdout:45 files removed 2024-09-18T00:54:26.023 INFO:teuthology.orchestra.run.smithi078.stdout:45 files removed 2024-09-18T00:54:26.047 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-radosgw 2024-09-18T00:54:26.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-radosgw 2024-09-18T00:54:26.604 INFO:teuthology.orchestra.run.smithi150.stdout:ceph packages for x86_64 307 kB/s | 84 kB 00:00 2024-09-18T00:54:26.611 INFO:teuthology.orchestra.run.smithi078.stdout:ceph packages for x86_64 302 kB/s | 84 kB 00:00 2024-09-18T00:54:26.871 INFO:teuthology.orchestra.run.smithi150.stdout:ceph noarch packages 53 kB/s | 12 kB 00:00 2024-09-18T00:54:26.904 INFO:teuthology.orchestra.run.smithi078.stdout:ceph noarch packages 47 kB/s | 12 kB 00:00 2024-09-18T00:54:27.147 INFO:teuthology.orchestra.run.smithi150.stdout:ceph source packages 7.9 kB/s | 1.9 kB 00:00 2024-09-18T00:54:27.170 INFO:teuthology.orchestra.run.smithi078.stdout:ceph source packages 8.2 kB/s | 1.9 kB 00:00 2024-09-18T00:54:29.729 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - BaseOS 3.3 MB/s | 8.3 MB 00:02 2024-09-18T00:54:29.758 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - BaseOS 3.2 MB/s | 8.3 MB 00:02 2024-09-18T00:54:31.688 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - AppStream 23 MB/s | 20 MB 00:00 2024-09-18T00:54:32.584 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - AppStream 12 MB/s | 20 MB 00:01 2024-09-18T00:54:36.826 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - CRB 15 MB/s | 6.5 MB 00:00 2024-09-18T00:54:37.764 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - CRB 2.8 MB/s | 6.5 MB 00:02 2024-09-18T00:54:38.469 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - Extras packages 70 kB/s | 19 kB 00:00 2024-09-18T00:54:39.630 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - Extras packages 35 kB/s | 19 kB 00:00 2024-09-18T00:54:40.048 INFO:teuthology.orchestra.run.smithi150.stdout:Extra Packages for Enterprise Linux 16 MB/s | 23 MB 00:01 2024-09-18T00:54:41.510 INFO:teuthology.orchestra.run.smithi078.stdout:Extra Packages for Enterprise Linux 13 MB/s | 23 MB 00:01 2024-09-18T00:54:45.495 INFO:teuthology.orchestra.run.smithi150.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-18T00:54:46.933 INFO:teuthology.orchestra.run.smithi078.stdout:lab-extras 36 kB/s | 1.7 kB 00:00 2024-09-18T00:54:47.757 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:54:47.763 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:54:47.763 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-09-18T00:54:47.763 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:54:47.763 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:54:47.763 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-radosgw x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 11 M 2024-09-18T00:54:47.763 INFO:teuthology.orchestra.run.smithi150.stdout:Upgrading: 2024-09-18T00:54:47.763 INFO:teuthology.orchestra.run.smithi150.stdout: librados2 x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 3.4 M 2024-09-18T00:54:47.764 INFO:teuthology.orchestra.run.smithi150.stdout: librbd1 x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 3.2 M 2024-09-18T00:54:47.764 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-09-18T00:54:47.764 INFO:teuthology.orchestra.run.smithi150.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-18T00:54:47.764 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-base x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 5.4 M 2024-09-18T00:54:47.764 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-common x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 21 M 2024-09-18T00:54:47.764 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-selinux x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 25 k 2024-09-18T00:54:47.764 INFO:teuthology.orchestra.run.smithi150.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-18T00:54:47.764 INFO:teuthology.orchestra.run.smithi150.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-18T00:54:47.764 INFO:teuthology.orchestra.run.smithi150.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-18T00:54:47.764 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs2 x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 751 k 2024-09-18T00:54:47.764 INFO:teuthology.orchestra.run.smithi150.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-18T00:54:47.765 INFO:teuthology.orchestra.run.smithi150.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-18T00:54:47.765 INFO:teuthology.orchestra.run.smithi150.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-18T00:54:47.765 INFO:teuthology.orchestra.run.smithi150.stdout: libradosstriper1 x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 506 k 2024-09-18T00:54:47.765 INFO:teuthology.orchestra.run.smithi150.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-18T00:54:47.765 INFO:teuthology.orchestra.run.smithi150.stdout: librgw2 x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 5.3 M 2024-09-18T00:54:47.765 INFO:teuthology.orchestra.run.smithi150.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-18T00:54:47.765 INFO:teuthology.orchestra.run.smithi150.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-18T00:54:47.765 INFO:teuthology.orchestra.run.smithi150.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-18T00:54:47.765 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-argparse x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 45 k 2024-09-18T00:54:47.765 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-common x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 136 k 2024-09-18T00:54:47.765 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cephfs x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 162 k 2024-09-18T00:54:47.766 INFO:teuthology.orchestra.run.smithi150.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-18T00:54:47.766 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rados x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 320 k 2024-09-18T00:54:47.766 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rbd x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 300 k 2024-09-18T00:54:47.766 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rgw x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 99 k 2024-09-18T00:54:47.766 INFO:teuthology.orchestra.run.smithi150.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-09-18T00:54:47.766 INFO:teuthology.orchestra.run.smithi150.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-09-18T00:54:47.766 INFO:teuthology.orchestra.run.smithi150.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-18T00:54:47.766 INFO:teuthology.orchestra.run.smithi150.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-18T00:54:47.766 INFO:teuthology.orchestra.run.smithi150.stdout:Installing weak dependencies: 2024-09-18T00:54:47.766 INFO:teuthology.orchestra.run.smithi150.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-09-18T00:54:47.766 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:54:47.767 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:54:47.767 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:54:47.767 INFO:teuthology.orchestra.run.smithi150.stdout:Install 30 Packages 2024-09-18T00:54:47.767 INFO:teuthology.orchestra.run.smithi150.stdout:Upgrade 2 Packages 2024-09-18T00:54:47.767 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:54:47.767 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 60 M 2024-09-18T00:54:47.767 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:54:48.430 INFO:teuthology.orchestra.run.smithi150.stdout:(1/32): ceph-base-19.1.1-330.gc2ddf972.el9.x86_ 12 MB/s | 5.4 MB 00:00 2024-09-18T00:54:48.488 INFO:teuthology.orchestra.run.smithi150.stdout:(2/32): ceph-selinux-19.1.1-330.gc2ddf972.el9.x 431 kB/s | 25 kB 00:00 2024-09-18T00:54:48.581 INFO:teuthology.orchestra.run.smithi150.stdout:(3/32): libcephfs2-19.1.1-330.gc2ddf972.el9.x86 8.0 MB/s | 751 kB 00:00 2024-09-18T00:54:48.714 INFO:teuthology.orchestra.run.smithi150.stdout:(4/32): ceph-radosgw-19.1.1-330.gc2ddf972.el9.x 14 MB/s | 11 MB 00:00 2024-09-18T00:54:48.739 INFO:teuthology.orchestra.run.smithi150.stdout:(5/32): libradosstriper1-19.1.1-330.gc2ddf972.e 3.1 MB/s | 506 kB 00:00 2024-09-18T00:54:48.798 INFO:teuthology.orchestra.run.smithi150.stdout:(6/32): python3-ceph-argparse-19.1.1-330.gc2ddf 774 kB/s | 45 kB 00:00 2024-09-18T00:54:48.873 INFO:teuthology.orchestra.run.smithi150.stdout:(7/32): python3-ceph-common-19.1.1-330.gc2ddf97 1.8 MB/s | 136 kB 00:00 2024-09-18T00:54:48.973 INFO:teuthology.orchestra.run.smithi150.stdout:(8/32): librgw2-19.1.1-330.gc2ddf972.el9.x86_64 21 MB/s | 5.3 MB 00:00 2024-09-18T00:54:48.990 INFO:teuthology.orchestra.run.smithi150.stdout:(9/32): python3-cephfs-19.1.1-330.gc2ddf972.el9 1.4 MB/s | 162 kB 00:00 2024-09-18T00:54:49.032 INFO:teuthology.orchestra.run.smithi150.stdout:(10/32): python3-rados-19.1.1-330.gc2ddf972.el9 5.3 MB/s | 320 kB 00:00 2024-09-18T00:54:49.057 INFO:teuthology.orchestra.run.smithi150.stdout:(11/32): python3-rbd-19.1.1-330.gc2ddf972.el9.x 4.4 MB/s | 300 kB 00:00 2024-09-18T00:54:49.083 INFO:teuthology.orchestra.run.smithi150.stdout:(12/32): python3-rgw-19.1.1-330.gc2ddf972.el9.x 1.9 MB/s | 99 kB 00:00 2024-09-18T00:54:49.142 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:54:49.154 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:54:49.154 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-18T00:54:49.154 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:54:49.154 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:54:49.154 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-radosgw x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 11 M 2024-09-18T00:54:49.154 INFO:teuthology.orchestra.run.smithi078.stdout:Upgrading: 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout: librados2 x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 3.4 M 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout: librbd1 x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 3.2 M 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-base x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 5.4 M 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-common x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 21 M 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-selinux x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 25 k 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs2 x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 751 k 2024-09-18T00:54:49.155 INFO:teuthology.orchestra.run.smithi078.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: libradosstriper1 x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 506 k 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: librgw2 x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 5.3 M 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-argparse x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 45 k 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-common x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 136 k 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cephfs x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 162 k 2024-09-18T00:54:49.156 INFO:teuthology.orchestra.run.smithi078.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-18T00:54:49.157 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rados x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 320 k 2024-09-18T00:54:49.157 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rbd x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 300 k 2024-09-18T00:54:49.157 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rgw x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 99 k 2024-09-18T00:54:49.157 INFO:teuthology.orchestra.run.smithi078.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-09-18T00:54:49.157 INFO:teuthology.orchestra.run.smithi078.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-09-18T00:54:49.157 INFO:teuthology.orchestra.run.smithi078.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-18T00:54:49.157 INFO:teuthology.orchestra.run.smithi078.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-18T00:54:49.157 INFO:teuthology.orchestra.run.smithi078.stdout:Installing weak dependencies: 2024-09-18T00:54:49.157 INFO:teuthology.orchestra.run.smithi078.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-09-18T00:54:49.157 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:54:49.157 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:54:49.158 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:54:49.158 INFO:teuthology.orchestra.run.smithi078.stdout:Install 30 Packages 2024-09-18T00:54:49.158 INFO:teuthology.orchestra.run.smithi078.stdout:Upgrade 2 Packages 2024-09-18T00:54:49.158 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:54:49.159 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 60 M 2024-09-18T00:54:49.159 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:54:49.283 INFO:teuthology.orchestra.run.smithi150.stdout:(13/32): ceph-common-19.1.1-330.gc2ddf972.el9.x 16 MB/s | 21 MB 00:01 2024-09-18T00:54:49.341 INFO:teuthology.orchestra.run.smithi150.stdout:(14/32): boost-program-options-1.75.0-8.el9.x86 377 kB/s | 107 kB 00:00 2024-09-18T00:54:49.366 INFO:teuthology.orchestra.run.smithi150.stdout:(15/32): libpmemobj-1.12.1-1.el9.x86_64.rpm 565 kB/s | 160 kB 00:00 2024-09-18T00:54:49.392 INFO:teuthology.orchestra.run.smithi150.stdout:(16/32): librabbitmq-0.11.0-7.el9.x86_64.rpm 417 kB/s | 45 kB 00:00 2024-09-18T00:54:49.434 INFO:teuthology.orchestra.run.smithi150.stdout:(17/32): librdkafka-1.6.1-102.el9.x86_64.rpm 7.0 MB/s | 662 kB 00:00 2024-09-18T00:54:49.459 INFO:teuthology.orchestra.run.smithi150.stdout:(18/32): python3-prettytable-0.7.2-27.el9.noarc 632 kB/s | 42 kB 00:00 2024-09-18T00:54:49.484 INFO:teuthology.orchestra.run.smithi150.stdout:(19/32): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.4 MB/s | 292 kB 00:00 2024-09-18T00:54:49.510 INFO:teuthology.orchestra.run.smithi150.stdout:(20/32): qatlib-service-24.02.0-1.el9.x86_64.rp 718 kB/s | 36 kB 00:00 2024-09-18T00:54:49.535 INFO:teuthology.orchestra.run.smithi150.stdout:(21/32): qatzip-libs-1.2.0-1.el9.x86_64.rpm 945 kB/s | 47 kB 00:00 2024-09-18T00:54:49.560 INFO:teuthology.orchestra.run.smithi150.stdout:(22/32): qatlib-24.02.0-1.el9.x86_64.rpm 1.7 MB/s | 221 kB 00:00 2024-09-18T00:54:49.627 INFO:teuthology.orchestra.run.smithi150.stdout:(23/32): libarrow-doc-9.0.0-11.el9.noarch.rpm 497 kB/s | 26 kB 00:00 2024-09-18T00:54:49.661 INFO:teuthology.orchestra.run.smithi150.stdout:(24/32): liboath-2.6.7-2.el9.x86_64.rpm 1.4 MB/s | 49 kB 00:00 2024-09-18T00:54:49.686 INFO:teuthology.orchestra.run.smithi150.stdout:(25/32): gperftools-libs-2.9.1-3.el9.x86_64.rpm 1.7 MB/s | 308 kB 00:00 2024-09-18T00:54:49.711 INFO:teuthology.orchestra.run.smithi150.stdout:(26/32): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-09-18T00:54:49.770 INFO:teuthology.orchestra.run.smithi150.stdout:(27/32): re2-20211101-20.el9.x86_64.rpm 3.2 MB/s | 191 kB 00:00 2024-09-18T00:54:49.887 INFO:teuthology.orchestra.run.smithi150.stdout:(28/32): parquet-libs-9.0.0-11.el9.x86_64.rpm 4.1 MB/s | 839 kB 00:00 2024-09-18T00:54:50.009 INFO:teuthology.orchestra.run.smithi078.stdout:(1/32): ceph-base-19.1.1-330.gc2ddf972.el9.x86_ 9.4 MB/s | 5.4 MB 00:00 2024-09-18T00:54:50.028 INFO:teuthology.orchestra.run.smithi150.stdout:(29/32): thrift-0.15.0-2.el9.x86_64.rpm 6.1 MB/s | 1.6 MB 00:00 2024-09-18T00:54:50.068 INFO:teuthology.orchestra.run.smithi078.stdout:(2/32): ceph-selinux-19.1.1-330.gc2ddf972.el9.x 432 kB/s | 25 kB 00:00 2024-09-18T00:54:50.162 INFO:teuthology.orchestra.run.smithi150.stdout:(30/32): librados2-19.1.1-330.gc2ddf972.el9.x86 12 MB/s | 3.4 MB 00:00 2024-09-18T00:54:50.184 INFO:teuthology.orchestra.run.smithi078.stdout:(3/32): libcephfs2-19.1.1-330.gc2ddf972.el9.x86 6.3 MB/s | 751 kB 00:00 2024-09-18T00:54:50.246 INFO:teuthology.orchestra.run.smithi150.stdout:(31/32): libarrow-9.0.0-11.el9.x86_64.rpm 6.2 MB/s | 4.4 MB 00:00 2024-09-18T00:54:50.252 INFO:teuthology.orchestra.run.smithi078.stdout:(4/32): libradosstriper1-19.1.1-330.gc2ddf972.e 7.4 MB/s | 506 kB 00:00 2024-09-18T00:54:50.496 INFO:teuthology.orchestra.run.smithi150.stdout:(32/32): librbd1-19.1.1-330.gc2ddf972.el9.x86_6 6.8 MB/s | 3.2 MB 00:00 2024-09-18T00:54:50.501 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:54:50.501 INFO:teuthology.orchestra.run.smithi150.stdout:Total 22 MB/s | 60 MB 00:02 2024-09-18T00:54:50.552 INFO:teuthology.orchestra.run.smithi078.stdout:(5/32): ceph-radosgw-19.1.1-330.gc2ddf972.el9.x 9.5 MB/s | 11 MB 00:01 2024-09-18T00:54:50.606 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:54:50.627 INFO:teuthology.orchestra.run.smithi078.stdout:(6/32): librgw2-19.1.1-330.gc2ddf972.el9.x86_64 14 MB/s | 5.3 MB 00:00 2024-09-18T00:54:50.641 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:54:50.641 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:54:50.652 INFO:teuthology.orchestra.run.smithi078.stdout:(7/32): python3-ceph-argparse-19.1.1-330.gc2ddf 449 kB/s | 45 kB 00:00 2024-09-18T00:54:50.853 INFO:teuthology.orchestra.run.smithi078.stdout:(8/32): ceph-common-19.1.1-330.gc2ddf972.el9.x8 15 MB/s | 21 MB 00:01 2024-09-18T00:54:50.878 INFO:teuthology.orchestra.run.smithi078.stdout:(9/32): python3-ceph-common-19.1.1-330.gc2ddf97 543 kB/s | 136 kB 00:00 2024-09-18T00:54:50.903 INFO:teuthology.orchestra.run.smithi078.stdout:(10/32): python3-cephfs-19.1.1-330.gc2ddf972.el 645 kB/s | 162 kB 00:00 2024-09-18T00:54:50.928 INFO:teuthology.orchestra.run.smithi078.stdout:(11/32): python3-rados-19.1.1-330.gc2ddf972.el9 4.2 MB/s | 320 kB 00:00 2024-09-18T00:54:50.962 INFO:teuthology.orchestra.run.smithi078.stdout:(12/32): python3-rbd-19.1.1-330.gc2ddf972.el9.x 3.5 MB/s | 300 kB 00:00 2024-09-18T00:54:50.987 INFO:teuthology.orchestra.run.smithi078.stdout:(13/32): python3-rgw-19.1.1-330.gc2ddf972.el9.x 1.2 MB/s | 99 kB 00:00 2024-09-18T00:54:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:(14/32): librabbitmq-0.11.0-7.el9.x86_64.rpm 319 kB/s | 45 kB 00:00 2024-09-18T00:54:51.154 INFO:teuthology.orchestra.run.smithi078.stdout:(15/32): boost-program-options-1.75.0-8.el9.x86 474 kB/s | 107 kB 00:00 2024-09-18T00:54:51.180 INFO:teuthology.orchestra.run.smithi078.stdout:(16/32): libpmemobj-1.12.1-1.el9.x86_64.rpm 737 kB/s | 160 kB 00:00 2024-09-18T00:54:51.230 INFO:teuthology.orchestra.run.smithi078.stdout:(17/32): python3-prettytable-0.7.2-27.el9.noarc 844 kB/s | 42 kB 00:00 2024-09-18T00:54:51.264 INFO:teuthology.orchestra.run.smithi078.stdout:(18/32): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.6 MB/s | 292 kB 00:00 2024-09-18T00:54:51.312 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:54:51.312 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:54:51.314 INFO:teuthology.orchestra.run.smithi078.stdout:(19/32): librdkafka-1.6.1-102.el9.x86_64.rpm 3.5 MB/s | 662 kB 00:00 2024-09-18T00:54:51.339 INFO:teuthology.orchestra.run.smithi078.stdout:(20/32): qatlib-24.02.0-1.el9.x86_64.rpm 2.0 MB/s | 221 kB 00:00 2024-09-18T00:54:51.364 INFO:teuthology.orchestra.run.smithi078.stdout:(21/32): qatlib-service-24.02.0-1.el9.x86_64.rp 357 kB/s | 36 kB 00:00 2024-09-18T00:54:51.389 INFO:teuthology.orchestra.run.smithi078.stdout:(22/32): qatzip-libs-1.2.0-1.el9.x86_64.rpm 630 kB/s | 47 kB 00:00 2024-09-18T00:54:51.457 INFO:teuthology.orchestra.run.smithi078.stdout:(23/32): libarrow-doc-9.0.0-11.el9.noarch.rpm 389 kB/s | 26 kB 00:00 2024-09-18T00:54:51.490 INFO:teuthology.orchestra.run.smithi078.stdout:(24/32): liboath-2.6.7-2.el9.x86_64.rpm 1.4 MB/s | 49 kB 00:00 2024-09-18T00:54:51.524 INFO:teuthology.orchestra.run.smithi078.stdout:(25/32): gperftools-libs-2.9.1-3.el9.x86_64.rpm 1.9 MB/s | 308 kB 00:00 2024-09-18T00:54:51.549 INFO:teuthology.orchestra.run.smithi078.stdout:(26/32): libunwind-1.6.2-1.el9.x86_64.rpm 1.1 MB/s | 67 kB 00:00 2024-09-18T00:54:51.599 INFO:teuthology.orchestra.run.smithi078.stdout:(27/32): re2-20211101-20.el9.x86_64.rpm 3.8 MB/s | 191 kB 00:00 2024-09-18T00:54:51.883 INFO:teuthology.orchestra.run.smithi078.stdout:(28/32): thrift-0.15.0-2.el9.x86_64.rpm 5.6 MB/s | 1.6 MB 00:00 2024-09-18T00:54:52.025 INFO:teuthology.orchestra.run.smithi078.stdout:(29/32): libarrow-9.0.0-11.el9.x86_64.rpm 6.7 MB/s | 4.4 MB 00:00 2024-09-18T00:54:52.158 INFO:teuthology.orchestra.run.smithi078.stdout:(30/32): librados2-19.1.1-330.gc2ddf972.el9.x86 12 MB/s | 3.4 MB 00:00 2024-09-18T00:54:52.242 INFO:teuthology.orchestra.run.smithi078.stdout:(31/32): librbd1-19.1.1-330.gc2ddf972.el9.x86_6 15 MB/s | 3.2 MB 00:00 2024-09-18T00:54:52.276 INFO:teuthology.orchestra.run.smithi078.stdout:(32/32): parquet-libs-9.0.0-11.el9.x86_64.rpm 1.1 MB/s | 839 kB 00:00 2024-09-18T00:54:52.281 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:54:52.281 INFO:teuthology.orchestra.run.smithi078.stdout:Total 19 MB/s | 60 MB 00:03 2024-09-18T00:54:52.383 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:54:52.419 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:54:52.419 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:54:52.963 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:54:53.045 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/34 2024-09-18T00:54:53.076 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/34 2024-09-18T00:54:53.093 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:54:53.093 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:54:53.101 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/34 2024-09-18T00:54:53.124 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-09-18T00:54:53.151 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-09-18T00:54:53.301 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 5/34 2024-09-18T00:54:53.355 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 5/34 2024-09-18T00:54:53.744 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : thrift-0.15.0-2.el9.x86_64 6/34 2024-09-18T00:54:53.803 INFO:teuthology.orchestra.run.smithi150.stdout: Upgrading : librados2-2:19.1.1-330.gc2ddf972.el9.x86_64 7/34 2024-09-18T00:54:53.895 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librados2-2:19.1.1-330.gc2ddf972.el9.x86_64 7/34 2024-09-18T00:54:53.982 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rados-2:19.1.1-330.gc2ddf972.el9.x86_64 8/34 2024-09-18T00:54:54.013 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libcephfs2-2:19.1.1-330.gc2ddf972.el9.x86_64 9/34 2024-09-18T00:54:54.059 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libcephfs2-2:19.1.1-330.gc2ddf972.el9.x86_64 9/34 2024-09-18T00:54:54.087 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : liboath-2.6.7-2.el9.x86_64 10/34 2024-09-18T00:54:54.131 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-ceph-argparse-2:19.1.1-330.gc2ddf972.el9.x 11/34 2024-09-18T00:54:54.200 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cephfs-2:19.1.1-330.gc2ddf972.el9.x86_64 12/34 2024-09-18T00:54:54.223 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libradosstriper1-2:19.1.1-330.gc2ddf972.el9.x86_64 13/34 2024-09-18T00:54:54.287 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libradosstriper1-2:19.1.1-330.gc2ddf972.el9.x86_64 13/34 2024-09-18T00:54:54.330 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 14/34 2024-09-18T00:54:54.438 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/34 2024-09-18T00:54:54.481 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 16/34 2024-09-18T00:54:54.522 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 17/34 2024-09-18T00:54:54.563 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 18/34 2024-09-18T00:54:54.592 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 19/34 2024-09-18T00:54:54.597 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:54:54.627 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 20/34 2024-09-18T00:54:54.678 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/34 2024-09-18T00:54:54.714 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/34 2024-09-18T00:54:54.739 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/34 2024-09-18T00:54:54.762 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-09-18T00:54:54.787 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-09-18T00:54:54.960 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 21/34 2024-09-18T00:54:54.981 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 5/34 2024-09-18T00:54:55.004 INFO:teuthology.orchestra.run.smithi150.stdout: Upgrading : librbd1-2:19.1.1-330.gc2ddf972.el9.x86_64 22/34 2024-09-18T00:54:55.033 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 5/34 2024-09-18T00:54:55.092 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librbd1-2:19.1.1-330.gc2ddf972.el9.x86_64 22/34 2024-09-18T00:54:55.118 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rbd-2:19.1.1-330.gc2ddf972.el9.x86_64 23/34 2024-09-18T00:54:55.162 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 24/34 2024-09-18T00:54:55.435 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : thrift-0.15.0-2.el9.x86_64 6/34 2024-09-18T00:54:55.499 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : librados2-2:19.1.1-330.gc2ddf972.el9.x86_64 7/34 2024-09-18T00:54:55.591 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librados2-2:19.1.1-330.gc2ddf972.el9.x86_64 7/34 2024-09-18T00:54:55.665 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 25/34 2024-09-18T00:54:55.686 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-rados-2:19.1.1-330.gc2ddf972.el9.x86_64 8/34 2024-09-18T00:54:55.690 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librgw2-2:19.1.1-330.gc2ddf972.el9.x86_64 26/34 2024-09-18T00:54:55.717 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libcephfs2-2:19.1.1-330.gc2ddf972.el9.x86_64 9/34 2024-09-18T00:54:55.763 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libcephfs2-2:19.1.1-330.gc2ddf972.el9.x86_64 9/34 2024-09-18T00:54:55.764 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librgw2-2:19.1.1-330.gc2ddf972.el9.x86_64 26/34 2024-09-18T00:54:55.800 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : liboath-2.6.7-2.el9.x86_64 10/34 2024-09-18T00:54:55.835 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-ceph-argparse-2:19.1.1-330.gc2ddf972.el9.x 11/34 2024-09-18T00:54:55.839 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rgw-2:19.1.1-330.gc2ddf972.el9.x86_64 27/34 2024-09-18T00:54:55.867 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-ceph-common-2:19.1.1-330.gc2ddf972.el9.x86 28/34 2024-09-18T00:54:55.904 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-cephfs-2:19.1.1-330.gc2ddf972.el9.x86_64 12/34 2024-09-18T00:54:55.927 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libradosstriper1-2:19.1.1-330.gc2ddf972.el9.x86_64 13/34 2024-09-18T00:54:55.991 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libradosstriper1-2:19.1.1-330.gc2ddf972.el9.x86_64 13/34 2024-09-18T00:54:56.026 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 14/34 2024-09-18T00:54:56.141 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/34 2024-09-18T00:54:56.177 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 16/34 2024-09-18T00:54:56.217 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 17/34 2024-09-18T00:54:56.250 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 18/34 2024-09-18T00:54:56.279 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 19/34 2024-09-18T00:54:56.317 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 20/34 2024-09-18T00:54:56.623 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 21/34 2024-09-18T00:54:56.666 INFO:teuthology.orchestra.run.smithi078.stdout: Upgrading : librbd1-2:19.1.1-330.gc2ddf972.el9.x86_64 22/34 2024-09-18T00:54:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librbd1-2:19.1.1-330.gc2ddf972.el9.x86_64 22/34 2024-09-18T00:54:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-rbd-2:19.1.1-330.gc2ddf972.el9.x86_64 23/34 2024-09-18T00:54:56.815 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 24/34 2024-09-18T00:54:57.293 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 25/34 2024-09-18T00:54:57.318 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : librgw2-2:19.1.1-330.gc2ddf972.el9.x86_64 26/34 2024-09-18T00:54:57.390 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librgw2-2:19.1.1-330.gc2ddf972.el9.x86_64 26/34 2024-09-18T00:54:57.466 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-rgw-2:19.1.1-330.gc2ddf972.el9.x86_64 27/34 2024-09-18T00:54:57.496 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-ceph-common-2:19.1.1-330.gc2ddf972.el9.x86 28/34 2024-09-18T00:54:58.192 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 29/34 2024-09-18T00:54:58.231 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 29/34 2024-09-18T00:54:58.783 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 29/34 2024-09-18T00:54:58.817 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-base-2:19.1.1-330.gc2ddf972.el9.x86_64 30/34 2024-09-18T00:54:58.900 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-base-2:19.1.1-330.gc2ddf972.el9.x86_64 30/34 2024-09-18T00:54:58.900 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-18T00:54:58.901 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-18T00:54:58.901 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:54:58.943 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-selinux-2:19.1.1-330.gc2ddf972.el9.x86_64 31/34 2024-09-18T00:54:59.790 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 29/34 2024-09-18T00:54:59.834 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 29/34 2024-09-18T00:55:00.417 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 29/34 2024-09-18T00:55:00.462 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-base-2:19.1.1-330.gc2ddf972.el9.x86_64 30/34 2024-09-18T00:55:00.565 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-base-2:19.1.1-330.gc2ddf972.el9.x86_64 30/34 2024-09-18T00:55:00.565 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-18T00:55:00.566 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-18T00:55:00.566 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:00.597 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-selinux-2:19.1.1-330.gc2ddf972.el9.x86_64 31/34 2024-09-18T00:55:08.897 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-selinux-2:19.1.1-330.gc2ddf972.el9.x86_64 31/34 2024-09-18T00:55:08.898 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /sys 2024-09-18T00:55:08.898 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /proc 2024-09-18T00:55:08.898 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /mnt 2024-09-18T00:55:08.898 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /var/tmp 2024-09-18T00:55:08.898 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /home 2024-09-18T00:55:08.898 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /root 2024-09-18T00:55:08.898 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /tmp 2024-09-18T00:55:08.898 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:09.902 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-radosgw-2:19.1.1-330.gc2ddf972.el9.x86_64 32/34 2024-09-18T00:55:09.927 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-radosgw-2:19.1.1-330.gc2ddf972.el9.x86_64 32/34 2024-09-18T00:55:09.928 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:09.928 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-18T00:55:09.928 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-18T00:55:09.928 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-18T00:55:09.928 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:09.930 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-09-18T00:55:10.021 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-09-18T00:55:10.021 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 34/34 2024-09-18T00:55:10.224 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-selinux-2:19.1.1-330.gc2ddf972.el9.x86_64 31/34 2024-09-18T00:55:10.224 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /sys 2024-09-18T00:55:10.224 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /proc 2024-09-18T00:55:10.224 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /mnt 2024-09-18T00:55:10.224 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /var/tmp 2024-09-18T00:55:10.224 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /home 2024-09-18T00:55:10.224 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /root 2024-09-18T00:55:10.225 INFO:teuthology.orchestra.run.smithi078.stdout:skipping the directory /tmp 2024-09-18T00:55:10.225 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:11.206 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-radosgw-2:19.1.1-330.gc2ddf972.el9.x86_64 32/34 2024-09-18T00:55:11.230 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-radosgw-2:19.1.1-330.gc2ddf972.el9.x86_64 32/34 2024-09-18T00:55:11.231 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:11.231 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-18T00:55:11.231 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-18T00:55:11.231 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-18T00:55:11.231 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:11.232 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-09-18T00:55:11.262 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 34/34 2024-09-18T00:55:11.262 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-base-2:19.1.1-330.gc2ddf972.el9.x86_64 1/34 2024-09-18T00:55:11.262 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 2/34 2024-09-18T00:55:11.262 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-radosgw-2:19.1.1-330.gc2ddf972.el9.x86_64 3/34 2024-09-18T00:55:11.262 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-selinux-2:19.1.1-330.gc2ddf972.el9.x86_64 4/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libcephfs2-2:19.1.1-330.gc2ddf972.el9.x86_64 5/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libradosstriper1-2:19.1.1-330.gc2ddf972.el9.x86_64 6/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librgw2-2:19.1.1-330.gc2ddf972.el9.x86_64 7/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-ceph-argparse-2:19.1.1-330.gc2ddf972.el9.x 8/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-ceph-common-2:19.1.1-330.gc2ddf972.el9.x86 9/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cephfs-2:19.1.1-330.gc2ddf972.el9.x86_64 10/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rados-2:19.1.1-330.gc2ddf972.el9.x86_64 11/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rbd-2:19.1.1-330.gc2ddf972.el9.x86_64 12/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rgw-2:19.1.1-330.gc2ddf972.el9.x86_64 13/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/34 2024-09-18T00:55:11.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/34 2024-09-18T00:55:11.264 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/34 2024-09-18T00:55:11.264 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/34 2024-09-18T00:55:11.264 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 20/34 2024-09-18T00:55:11.264 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 21/34 2024-09-18T00:55:11.264 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 22/34 2024-09-18T00:55:11.264 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 23/34 2024-09-18T00:55:11.264 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 24/34 2024-09-18T00:55:11.264 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 25/34 2024-09-18T00:55:11.264 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 26/34 2024-09-18T00:55:11.265 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 27/34 2024-09-18T00:55:11.265 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 28/34 2024-09-18T00:55:11.265 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : re2-1:20211101-20.el9.x86_64 29/34 2024-09-18T00:55:11.265 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 30/34 2024-09-18T00:55:11.265 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librados2-2:19.1.1-330.gc2ddf972.el9.x86_64 31/34 2024-09-18T00:55:11.265 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 32/34 2024-09-18T00:55:11.265 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librbd1-2:19.1.1-330.gc2ddf972.el9.x86_64 33/34 2024-09-18T00:55:11.317 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-09-18T00:55:11.317 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 34/34 2024-09-18T00:55:12.305 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 34/34 2024-09-18T00:55:12.305 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout:Upgraded: 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout: librados2-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout: librbd1-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-base-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-radosgw-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-selinux-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-18T00:55:12.306 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs2-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: libradosstriper1-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: librgw2-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-argparse-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cephfs-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.307 INFO:teuthology.orchestra.run.smithi150.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-18T00:55:12.308 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rados-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.308 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rbd-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.308 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rgw-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:12.308 INFO:teuthology.orchestra.run.smithi150.stdout: qatlib-24.02.0-1.el9.x86_64 2024-09-18T00:55:12.308 INFO:teuthology.orchestra.run.smithi150.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-09-18T00:55:12.308 INFO:teuthology.orchestra.run.smithi150.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-09-18T00:55:12.308 INFO:teuthology.orchestra.run.smithi150.stdout: re2-1:20211101-20.el9.x86_64 2024-09-18T00:55:12.308 INFO:teuthology.orchestra.run.smithi150.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-18T00:55:12.309 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:12.309 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:55:12.548 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 34/34 2024-09-18T00:55:12.548 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-base-2:19.1.1-330.gc2ddf972.el9.x86_64 1/34 2024-09-18T00:55:12.548 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 2/34 2024-09-18T00:55:12.548 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-radosgw-2:19.1.1-330.gc2ddf972.el9.x86_64 3/34 2024-09-18T00:55:12.548 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-selinux-2:19.1.1-330.gc2ddf972.el9.x86_64 4/34 2024-09-18T00:55:12.548 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephfs2-2:19.1.1-330.gc2ddf972.el9.x86_64 5/34 2024-09-18T00:55:12.549 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libradosstriper1-2:19.1.1-330.gc2ddf972.el9.x86_64 6/34 2024-09-18T00:55:12.549 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librgw2-2:19.1.1-330.gc2ddf972.el9.x86_64 7/34 2024-09-18T00:55:12.549 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-ceph-argparse-2:19.1.1-330.gc2ddf972.el9.x 8/34 2024-09-18T00:55:12.549 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-ceph-common-2:19.1.1-330.gc2ddf972.el9.x86 9/34 2024-09-18T00:55:12.549 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cephfs-2:19.1.1-330.gc2ddf972.el9.x86_64 10/34 2024-09-18T00:55:12.549 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rados-2:19.1.1-330.gc2ddf972.el9.x86_64 11/34 2024-09-18T00:55:12.549 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rbd-2:19.1.1-330.gc2ddf972.el9.x86_64 12/34 2024-09-18T00:55:12.549 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rgw-2:19.1.1-330.gc2ddf972.el9.x86_64 13/34 2024-09-18T00:55:12.549 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/34 2024-09-18T00:55:12.549 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/34 2024-09-18T00:55:12.550 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/34 2024-09-18T00:55:12.550 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/34 2024-09-18T00:55:12.550 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/34 2024-09-18T00:55:12.550 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/34 2024-09-18T00:55:12.550 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 20/34 2024-09-18T00:55:12.550 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 21/34 2024-09-18T00:55:12.550 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 22/34 2024-09-18T00:55:12.550 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 23/34 2024-09-18T00:55:12.550 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 24/34 2024-09-18T00:55:12.550 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 25/34 2024-09-18T00:55:12.550 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 26/34 2024-09-18T00:55:12.551 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 27/34 2024-09-18T00:55:12.551 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 28/34 2024-09-18T00:55:12.551 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : re2-1:20211101-20.el9.x86_64 29/34 2024-09-18T00:55:12.551 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 30/34 2024-09-18T00:55:12.551 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librados2-2:19.1.1-330.gc2ddf972.el9.x86_64 31/34 2024-09-18T00:55:12.552 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 32/34 2024-09-18T00:55:12.552 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librbd1-2:19.1.1-330.gc2ddf972.el9.x86_64 33/34 2024-09-18T00:55:12.720 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-test 2024-09-18T00:55:13.302 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:28 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:55:13.443 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:55:13.444 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:13.444 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-09-18T00:55:13.444 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:13.444 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:55:13.444 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-test x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 49 M 2024-09-18T00:55:13.444 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-09-18T00:55:13.444 INFO:teuthology.orchestra.run.smithi150.stdout: libcephsqlite x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 163 k 2024-09-18T00:55:13.444 INFO:teuthology.orchestra.run.smithi150.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-18T00:55:13.444 INFO:teuthology.orchestra.run.smithi150.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-18T00:55:13.444 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:13.445 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:55:13.445 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:13.445 INFO:teuthology.orchestra.run.smithi150.stdout:Install 4 Packages 2024-09-18T00:55:13.445 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:13.445 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 49 M 2024-09-18T00:55:13.445 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 209 M 2024-09-18T00:55:13.445 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:55:13.708 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 34/34 2024-09-18T00:55:13.708 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:13.708 INFO:teuthology.orchestra.run.smithi078.stdout:Upgraded: 2024-09-18T00:55:13.708 INFO:teuthology.orchestra.run.smithi078.stdout: librados2-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.708 INFO:teuthology.orchestra.run.smithi078.stdout: librbd1-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.708 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:55:13.708 INFO:teuthology.orchestra.run.smithi078.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-18T00:55:13.708 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-base-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.708 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.708 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-radosgw-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.708 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-selinux-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.709 INFO:teuthology.orchestra.run.smithi078.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-18T00:55:13.709 INFO:teuthology.orchestra.run.smithi078.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-18T00:55:13.709 INFO:teuthology.orchestra.run.smithi078.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-18T00:55:13.709 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs2-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.709 INFO:teuthology.orchestra.run.smithi078.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-18T00:55:13.709 INFO:teuthology.orchestra.run.smithi078.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-18T00:55:13.709 INFO:teuthology.orchestra.run.smithi078.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-18T00:55:13.709 INFO:teuthology.orchestra.run.smithi078.stdout: libradosstriper1-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.709 INFO:teuthology.orchestra.run.smithi078.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-18T00:55:13.709 INFO:teuthology.orchestra.run.smithi078.stdout: librgw2-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.710 INFO:teuthology.orchestra.run.smithi078.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-18T00:55:13.710 INFO:teuthology.orchestra.run.smithi078.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-18T00:55:13.710 INFO:teuthology.orchestra.run.smithi078.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-18T00:55:13.710 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-argparse-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.710 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-common-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.710 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cephfs-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.710 INFO:teuthology.orchestra.run.smithi078.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-18T00:55:13.710 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rados-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.710 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rbd-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.710 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rgw-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:13.711 INFO:teuthology.orchestra.run.smithi078.stdout: qatlib-24.02.0-1.el9.x86_64 2024-09-18T00:55:13.711 INFO:teuthology.orchestra.run.smithi078.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-09-18T00:55:13.711 INFO:teuthology.orchestra.run.smithi078.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-09-18T00:55:13.711 INFO:teuthology.orchestra.run.smithi078.stdout: re2-1:20211101-20.el9.x86_64 2024-09-18T00:55:13.711 INFO:teuthology.orchestra.run.smithi078.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-18T00:55:13.711 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:13.711 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:55:13.717 INFO:teuthology.orchestra.run.smithi150.stdout:(1/4): libcephsqlite-19.1.1-330.gc2ddf972.el9.x 724 kB/s | 163 kB 00:00 2024-09-18T00:55:14.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-test 2024-09-18T00:55:14.326 INFO:teuthology.orchestra.run.smithi150.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 105 kB/s | 64 kB 00:00 2024-09-18T00:55:14.401 INFO:teuthology.orchestra.run.smithi150.stdout:(3/4): socat-1.7.4.1-6.el9.x86_64.rpm 334 kB/s | 304 kB 00:00 2024-09-18T00:55:14.678 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:28 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:55:14.815 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:55:14.815 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:14.815 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-18T00:55:14.815 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:14.815 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:55:14.816 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-test x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 49 M 2024-09-18T00:55:14.816 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-18T00:55:14.816 INFO:teuthology.orchestra.run.smithi078.stdout: libcephsqlite x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 163 k 2024-09-18T00:55:14.816 INFO:teuthology.orchestra.run.smithi078.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-18T00:55:14.816 INFO:teuthology.orchestra.run.smithi078.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-18T00:55:14.816 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:14.816 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:55:14.816 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:14.816 INFO:teuthology.orchestra.run.smithi078.stdout:Install 4 Packages 2024-09-18T00:55:14.816 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:14.817 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 49 M 2024-09-18T00:55:14.817 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 209 M 2024-09-18T00:55:14.817 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:55:14.993 INFO:teuthology.orchestra.run.smithi150.stdout:(4/4): ceph-test-19.1.1-330.gc2ddf972.el9.x86_6 32 MB/s | 49 MB 00:01 2024-09-18T00:55:14.996 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:14.996 INFO:teuthology.orchestra.run.smithi150.stdout:Total 32 MB/s | 49 MB 00:01 2024-09-18T00:55:15.024 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:55:15.045 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:55:15.045 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:55:15.088 INFO:teuthology.orchestra.run.smithi078.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.3 MB/s | 304 kB 00:00 2024-09-18T00:55:15.114 INFO:teuthology.orchestra.run.smithi078.stdout:(2/4): libcephsqlite-19.1.1-330.gc2ddf972.el9.x 658 kB/s | 163 kB 00:00 2024-09-18T00:55:15.147 INFO:teuthology.orchestra.run.smithi078.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.1 MB/s | 64 kB 00:00 2024-09-18T00:55:15.440 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:55:15.440 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:55:15.965 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:55:16.056 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-18T00:55:16.109 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-18T00:55:16.137 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libcephsqlite-2:19.1.1-330.gc2ddf972.el9.x86_64 3/4 2024-09-18T00:55:16.372 INFO:teuthology.orchestra.run.smithi078.stdout:(4/4): ceph-test-19.1.1-330.gc2ddf972.el9.x86_6 32 MB/s | 49 MB 00:01 2024-09-18T00:55:16.374 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:16.375 INFO:teuthology.orchestra.run.smithi078.stdout:Total 32 MB/s | 49 MB 00:01 2024-09-18T00:55:16.401 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:55:16.421 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:55:16.421 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:55:16.808 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:55:16.808 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:55:17.347 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:55:17.430 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-18T00:55:17.472 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-18T00:55:17.499 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libcephsqlite-2:19.1.1-330.gc2ddf972.el9.x86_64 3/4 2024-09-18T00:55:20.790 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libcephsqlite-2:19.1.1-330.gc2ddf972.el9.x86_64 3/4 2024-09-18T00:55:20.878 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-test-2:19.1.1-330.gc2ddf972.el9.x86_64 4/4 2024-09-18T00:55:22.117 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libcephsqlite-2:19.1.1-330.gc2ddf972.el9.x86_64 3/4 2024-09-18T00:55:22.206 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-test-2:19.1.1-330.gc2ddf972.el9.x86_64 4/4 2024-09-18T00:55:23.193 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-test-2:19.1.1-330.gc2ddf972.el9.x86_64 4/4 2024-09-18T00:55:23.193 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-test-2:19.1.1-330.gc2ddf972.el9.x86_64 1/4 2024-09-18T00:55:23.193 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libcephsqlite-2:19.1.1-330.gc2ddf972.el9.x86_64 2/4 2024-09-18T00:55:23.193 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-18T00:55:23.518 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-18T00:55:23.518 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:23.518 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:55:23.518 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-test-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:23.518 INFO:teuthology.orchestra.run.smithi150.stdout: libcephsqlite-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:23.518 INFO:teuthology.orchestra.run.smithi150.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-18T00:55:23.518 INFO:teuthology.orchestra.run.smithi150.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-18T00:55:23.518 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:23.518 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:55:23.821 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph 2024-09-18T00:55:24.414 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:39 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:55:24.559 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:55:24.562 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:24.562 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-09-18T00:55:24.562 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:24.562 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:55:24.562 INFO:teuthology.orchestra.run.smithi150.stdout: ceph x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 6.5 k 2024-09-18T00:55:24.562 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-09-18T00:55:24.562 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mds x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 2.4 M 2024-09-18T00:55:24.562 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 1.5 M 2024-09-18T00:55:24.562 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-modules-core noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 249 k 2024-09-18T00:55:24.562 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mon x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 4.7 M 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-osd x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 17 M 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-18T00:55:24.563 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-18T00:55:24.564 INFO:teuthology.orchestra.run.smithi150.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-18T00:55:24.565 INFO:teuthology.orchestra.run.smithi150.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-18T00:55:24.565 INFO:teuthology.orchestra.run.smithi150.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-18T00:55:24.565 INFO:teuthology.orchestra.run.smithi150.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-18T00:55:24.565 INFO:teuthology.orchestra.run.smithi150.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-18T00:55:24.565 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:24.565 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:55:24.565 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:24.565 INFO:teuthology.orchestra.run.smithi150.stdout:Install 37 Packages 2024-09-18T00:55:24.565 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:24.566 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 30 M 2024-09-18T00:55:24.566 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 108 M 2024-09-18T00:55:24.566 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:55:24.602 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-test-2:19.1.1-330.gc2ddf972.el9.x86_64 4/4 2024-09-18T00:55:24.602 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-test-2:19.1.1-330.gc2ddf972.el9.x86_64 1/4 2024-09-18T00:55:24.603 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephsqlite-2:19.1.1-330.gc2ddf972.el9.x86_64 2/4 2024-09-18T00:55:24.603 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-18T00:55:24.905 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-18T00:55:24.905 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:24.905 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:55:24.905 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-test-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:24.905 INFO:teuthology.orchestra.run.smithi078.stdout: libcephsqlite-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:24.906 INFO:teuthology.orchestra.run.smithi078.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-18T00:55:24.906 INFO:teuthology.orchestra.run.smithi078.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-18T00:55:24.906 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:24.906 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:55:25.177 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph 2024-09-18T00:55:25.302 INFO:teuthology.orchestra.run.smithi150.stdout:(1/37): ceph-19.1.1-330.gc2ddf972.el9.x86_64.rp 35 kB/s | 6.5 kB 00:00 2024-09-18T00:55:25.486 INFO:teuthology.orchestra.run.smithi150.stdout:(2/37): ceph-mds-19.1.1-330.gc2ddf972.el9.x86_6 6.4 MB/s | 2.4 MB 00:00 2024-09-18T00:55:25.528 INFO:teuthology.orchestra.run.smithi150.stdout:(3/37): ceph-mgr-19.1.1-330.gc2ddf972.el9.x86_6 3.7 MB/s | 1.5 MB 00:00 2024-09-18T00:55:25.628 INFO:teuthology.orchestra.run.smithi150.stdout:(4/37): ceph-mgr-modules-core-19.1.1-330.gc2ddf 2.4 MB/s | 249 kB 00:00 2024-09-18T00:55:25.773 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:39 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:55:25.845 INFO:teuthology.orchestra.run.smithi150.stdout:(5/37): ceph-mon-19.1.1-330.gc2ddf972.el9.x86_6 8.7 MB/s | 4.7 MB 00:00 2024-09-18T00:55:25.870 INFO:teuthology.orchestra.run.smithi150.stdout:(6/37): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.0 MB/s | 253 kB 00:00 2024-09-18T00:55:25.916 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:55:25.919 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout: ceph x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 6.5 k 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mds x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 2.4 M 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 1.5 M 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-modules-core noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 249 k 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mon x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 4.7 M 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-osd x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 17 M 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-09-18T00:55:25.920 INFO:teuthology.orchestra.run.smithi078.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-18T00:55:25.921 INFO:teuthology.orchestra.run.smithi078.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-18T00:55:25.922 INFO:teuthology.orchestra.run.smithi078.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-18T00:55:25.923 INFO:teuthology.orchestra.run.smithi078.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-18T00:55:25.923 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:25.923 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:55:25.923 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:25.923 INFO:teuthology.orchestra.run.smithi078.stdout:Install 37 Packages 2024-09-18T00:55:25.923 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:25.923 INFO:teuthology.orchestra.run.smithi150.stdout:(7/37): python3-ply-3.11-14.el9.noarch.rpm 2.1 MB/s | 106 kB 00:00 2024-09-18T00:55:25.924 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 30 M 2024-09-18T00:55:25.924 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 108 M 2024-09-18T00:55:25.924 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:55:26.105 INFO:teuthology.orchestra.run.smithi150.stdout:(8/37): ceph-osd-19.1.1-330.gc2ddf972.el9.x86_6 27 MB/s | 17 MB 00:00 2024-09-18T00:55:26.130 INFO:teuthology.orchestra.run.smithi150.stdout:(9/37): python3-pycparser-2.20-6.el9.noarch.rpm 648 kB/s | 135 kB 00:00 2024-09-18T00:55:26.188 INFO:teuthology.orchestra.run.smithi150.stdout:(10/37): python3-urllib3-1.26.5-6.el9.noarch.rp 3.6 MB/s | 215 kB 00:00 2024-09-18T00:55:26.289 INFO:teuthology.orchestra.run.smithi150.stdout:(11/37): python3-cryptography-36.0.1-4.el9.x86_ 2.8 MB/s | 1.2 MB 00:00 2024-09-18T00:55:26.330 INFO:teuthology.orchestra.run.smithi150.stdout:(12/37): python3-requests-2.25.1-8.el9.noarch.r 554 kB/s | 125 kB 00:00 2024-09-18T00:55:26.356 INFO:teuthology.orchestra.run.smithi150.stdout:(13/37): lua-5.4.4-4.el9.x86_64.rpm 1.1 MB/s | 188 kB 00:00 2024-09-18T00:55:26.398 INFO:teuthology.orchestra.run.smithi150.stdout:(14/37): python3-packaging-20.9-5.el9.noarch.rp 1.8 MB/s | 77 kB 00:00 2024-09-18T00:55:26.431 INFO:teuthology.orchestra.run.smithi150.stdout:(15/37): python3-markupsafe-1.1.1-12.el9.x86_64 345 kB/s | 35 kB 00:00 2024-09-18T00:55:26.457 INFO:teuthology.orchestra.run.smithi150.stdout:(16/37): python3-pytz-2021.1-5.el9.noarch.rpm 871 kB/s | 51 kB 00:00 2024-09-18T00:55:26.485 INFO:teuthology.orchestra.run.smithi150.stdout:(17/37): python3-mako-1.1.4-6.el9.noarch.rpm 878 kB/s | 172 kB 00:00 2024-09-18T00:55:26.502 INFO:teuthology.orchestra.run.smithi150.stdout:(18/37): python3-toml-0.10.2-6.el9.noarch.rpm 593 kB/s | 42 kB 00:00 2024-09-18T00:55:26.544 INFO:teuthology.orchestra.run.smithi150.stdout:(19/37): lua-devel-5.4.4-4.el9.x86_64.rpm 256 kB/s | 22 kB 00:00 2024-09-18T00:55:26.561 INFO:teuthology.orchestra.run.smithi150.stdout:(20/37): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 741 kB/s | 43 kB 00:00 2024-09-18T00:55:26.594 INFO:teuthology.orchestra.run.smithi150.stdout:(21/37): luarocks-3.9.2-5.el9.noarch.rpm 1.4 MB/s | 151 kB 00:00 2024-09-18T00:55:26.620 INFO:teuthology.orchestra.run.smithi150.stdout:(22/37): python3-jaraco-8.2.1-3.el9.noarch.rpm 462 kB/s | 11 kB 00:00 2024-09-18T00:55:26.637 INFO:teuthology.orchestra.run.smithi150.stdout:(23/37): python3-jaraco-classes-3.2.1-5.el9.noa 1.0 MB/s | 18 kB 00:00 2024-09-18T00:55:26.662 INFO:teuthology.orchestra.run.smithi150.stdout:(24/37): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.4 MB/s | 172 kB 00:00 2024-09-18T00:55:26.679 INFO:teuthology.orchestra.run.smithi150.stdout:(25/37): python3-jaraco-collections-3.0.0-8.el9 553 kB/s | 23 kB 00:00 2024-09-18T00:55:26.704 INFO:teuthology.orchestra.run.smithi150.stdout:(26/37): python3-cherrypy-18.6.1-2.el9.noarch.r 2.6 MB/s | 358 kB 00:00 2024-09-18T00:55:26.721 INFO:teuthology.orchestra.run.smithi150.stdout:(27/37): python3-jaraco-functools-3.5.0-2.el9.n 329 kB/s | 19 kB 00:00 2024-09-18T00:55:26.738 INFO:teuthology.orchestra.run.smithi150.stdout:(28/37): python3-jaraco-text-3.2.0-6.el9.noarch 332 kB/s | 20 kB 00:00 2024-09-18T00:55:26.755 INFO:teuthology.orchestra.run.smithi150.stdout:(29/37): python3-logutils-0.3.5-21.el9.noarch.r 910 kB/s | 46 kB 00:00 2024-09-18T00:55:26.780 INFO:teuthology.orchestra.run.smithi150.stdout:(30/37): python3-more-itertools-8.12.0-2.el9.no 1.3 MB/s | 79 kB 00:00 2024-09-18T00:55:26.797 INFO:teuthology.orchestra.run.smithi150.stdout:(31/37): python3-portend-3.1.0-2.el9.noarch.rpm 393 kB/s | 16 kB 00:00 2024-09-18T00:55:26.822 INFO:teuthology.orchestra.run.smithi150.stdout:(32/37): python3-pyOpenSSL-21.0.0-1.el9.noarch. 2.1 MB/s | 90 kB 00:00 2024-09-18T00:55:26.839 INFO:teuthology.orchestra.run.smithi150.stdout:(33/37): python3-tempora-5.0.0-2.el9.noarch.rpm 857 kB/s | 36 kB 00:00 2024-09-18T00:55:26.916 INFO:teuthology.orchestra.run.smithi150.stdout:(34/37): python3-pecan-1.4.2-3.el9.noarch.rpm 1.5 MB/s | 272 kB 00:00 2024-09-18T00:55:26.940 INFO:teuthology.orchestra.run.smithi150.stdout:(35/37): python3-zc-lockfile-2.0-10.el9.noarch. 852 kB/s | 20 kB 00:00 2024-09-18T00:55:26.966 INFO:teuthology.orchestra.run.smithi150.stdout:(36/37): python3-webob-1.8.8-2.el9.noarch.rpm 1.6 MB/s | 230 kB 00:00 2024-09-18T00:55:26.991 INFO:teuthology.orchestra.run.smithi150.stdout:(37/37): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.8 MB/s | 427 kB 00:00 2024-09-18T00:55:26.994 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:26.994 INFO:teuthology.orchestra.run.smithi150.stdout:Total 12 MB/s | 30 MB 00:02 2024-09-18T00:55:27.140 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:55:27.180 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:55:27.181 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:55:27.641 INFO:teuthology.orchestra.run.smithi078.stdout:(1/37): ceph-19.1.1-330.gc2ddf972.el9.x86_64.rp 48 kB/s | 6.5 kB 00:00 2024-09-18T00:55:27.684 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:55:27.684 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:55:27.824 INFO:teuthology.orchestra.run.smithi078.stdout:(2/37): ceph-mgr-19.1.1-330.gc2ddf972.el9.x86_6 4.8 MB/s | 1.5 MB 00:00 2024-09-18T00:55:27.882 INFO:teuthology.orchestra.run.smithi078.stdout:(3/37): ceph-mds-19.1.1-330.gc2ddf972.el9.x86_6 6.3 MB/s | 2.4 MB 00:00 2024-09-18T00:55:27.958 INFO:teuthology.orchestra.run.smithi078.stdout:(4/37): ceph-mgr-modules-core-19.1.1-330.gc2ddf 3.3 MB/s | 249 kB 00:00 2024-09-18T00:55:28.149 INFO:teuthology.orchestra.run.smithi078.stdout:(5/37): ceph-mon-19.1.1-330.gc2ddf972.el9.x86_6 9.3 MB/s | 4.7 MB 00:00 2024-09-18T00:55:28.175 INFO:teuthology.orchestra.run.smithi078.stdout:(6/37): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.1 MB/s | 253 kB 00:00 2024-09-18T00:55:28.225 INFO:teuthology.orchestra.run.smithi078.stdout:(7/37): python3-ply-3.11-14.el9.noarch.rpm 2.1 MB/s | 106 kB 00:00 2024-09-18T00:55:28.392 INFO:teuthology.orchestra.run.smithi078.stdout:(8/37): ceph-osd-19.1.1-330.gc2ddf972.el9.x86_6 30 MB/s | 17 MB 00:00 2024-09-18T00:55:28.417 INFO:teuthology.orchestra.run.smithi078.stdout:(9/37): python3-pycparser-2.20-6.el9.noarch.rpm 704 kB/s | 135 kB 00:00 2024-09-18T00:55:28.501 INFO:teuthology.orchestra.run.smithi078.stdout:(10/37): python3-urllib3-1.26.5-6.el9.noarch.rp 2.5 MB/s | 215 kB 00:00 2024-09-18T00:55:28.551 INFO:teuthology.orchestra.run.smithi078.stdout:(11/37): python3-requests-2.25.1-8.el9.noarch.r 788 kB/s | 125 kB 00:00 2024-09-18T00:55:28.652 INFO:teuthology.orchestra.run.smithi078.stdout:(12/37): lua-5.4.4-4.el9.x86_64.rpm 1.2 MB/s | 188 kB 00:00 2024-09-18T00:55:28.701 INFO:teuthology.orchestra.run.smithi078.stdout:(13/37): python3-markupsafe-1.1.1-12.el9.x86_64 699 kB/s | 35 kB 00:00 2024-09-18T00:55:28.735 INFO:teuthology.orchestra.run.smithi078.stdout:(14/37): python3-mako-1.1.4-6.el9.noarch.rpm 936 kB/s | 172 kB 00:00 2024-09-18T00:55:28.777 INFO:teuthology.orchestra.run.smithi078.stdout:(15/37): python3-cryptography-36.0.1-4.el9.x86_ 2.0 MB/s | 1.2 MB 00:00 2024-09-18T00:55:28.794 INFO:teuthology.orchestra.run.smithi078.stdout:(16/37): python3-packaging-20.9-5.el9.noarch.rp 835 kB/s | 77 kB 00:00 2024-09-18T00:55:28.811 INFO:teuthology.orchestra.run.smithi078.stdout:(17/37): python3-pytz-2021.1-5.el9.noarch.rpm 677 kB/s | 51 kB 00:00 2024-09-18T00:55:28.828 INFO:teuthology.orchestra.run.smithi078.stdout:(18/37): python3-toml-0.10.2-6.el9.noarch.rpm 827 kB/s | 42 kB 00:00 2024-09-18T00:55:28.861 INFO:teuthology.orchestra.run.smithi078.stdout:(19/37): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.3 MB/s | 43 kB 00:00 2024-09-18T00:55:28.878 INFO:teuthology.orchestra.run.smithi078.stdout:(20/37): lua-devel-5.4.4-4.el9.x86_64.rpm 265 kB/s | 22 kB 00:00 2024-09-18T00:55:28.912 INFO:teuthology.orchestra.run.smithi078.stdout:(21/37): luarocks-3.9.2-5.el9.noarch.rpm 1.5 MB/s | 151 kB 00:00 2024-09-18T00:55:28.937 INFO:teuthology.orchestra.run.smithi078.stdout:(22/37): python3-cheroot-8.6.0-4.el9.noarch.rpm 2.3 MB/s | 172 kB 00:00 2024-09-18T00:55:28.963 INFO:teuthology.orchestra.run.smithi078.stdout:(23/37): python3-jaraco-8.2.1-3.el9.noarch.rpm 221 kB/s | 11 kB 00:00 2024-09-18T00:55:28.988 INFO:teuthology.orchestra.run.smithi078.stdout:(24/37): python3-jaraco-classes-3.2.1-5.el9.noa 353 kB/s | 18 kB 00:00 2024-09-18T00:55:29.013 INFO:teuthology.orchestra.run.smithi078.stdout:(25/37): python3-jaraco-collections-3.0.0-8.el9 461 kB/s | 23 kB 00:00 2024-09-18T00:55:29.038 INFO:teuthology.orchestra.run.smithi078.stdout:(26/37): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-09-18T00:55:29.064 INFO:teuthology.orchestra.run.smithi078.stdout:(27/37): python3-jaraco-text-3.2.0-6.el9.noarch 392 kB/s | 20 kB 00:00 2024-09-18T00:55:29.070 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:55:29.089 INFO:teuthology.orchestra.run.smithi078.stdout:(28/37): python3-logutils-0.3.5-21.el9.noarch.r 921 kB/s | 46 kB 00:00 2024-09-18T00:55:29.123 INFO:teuthology.orchestra.run.smithi078.stdout:(29/37): python3-more-itertools-8.12.0-2.el9.no 1.3 MB/s | 79 kB 00:00 2024-09-18T00:55:29.127 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/37 2024-09-18T00:55:29.156 INFO:teuthology.orchestra.run.smithi078.stdout:(30/37): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-09-18T00:55:29.159 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/37 2024-09-18T00:55:29.181 INFO:teuthology.orchestra.run.smithi078.stdout:(31/37): python3-portend-3.1.0-2.el9.noarch.rpm 281 kB/s | 16 kB 00:00 2024-09-18T00:55:29.187 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/37 2024-09-18T00:55:29.215 INFO:teuthology.orchestra.run.smithi078.stdout:(32/37): python3-pecan-1.4.2-3.el9.noarch.rpm 2.1 MB/s | 272 kB 00:00 2024-09-18T00:55:29.240 INFO:teuthology.orchestra.run.smithi078.stdout:(33/37): python3-tempora-5.0.0-2.el9.noarch.rpm 699 kB/s | 36 kB 00:00 2024-09-18T00:55:29.259 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 4/37 2024-09-18T00:55:29.265 INFO:teuthology.orchestra.run.smithi078.stdout:(34/37): python3-pyOpenSSL-21.0.0-1.el9.noarch. 832 kB/s | 90 kB 00:00 2024-09-18T00:55:29.291 INFO:teuthology.orchestra.run.smithi078.stdout:(35/37): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-18T00:55:29.316 INFO:teuthology.orchestra.run.smithi078.stdout:(36/37): python3-zc-lockfile-2.0-10.el9.noarch. 397 kB/s | 20 kB 00:00 2024-09-18T00:55:29.328 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : luarocks-3.9.2-5.el9.noarch 5/37 2024-09-18T00:55:29.350 INFO:teuthology.orchestra.run.smithi078.stdout:(37/37): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.8 MB/s | 427 kB 00:00 2024-09-18T00:55:29.354 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:29.354 INFO:teuthology.orchestra.run.smithi078.stdout:Total 8.8 MB/s | 30 MB 00:03 2024-09-18T00:55:29.361 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 6/37 2024-09-18T00:55:29.454 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 7/37 2024-09-18T00:55:29.510 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:55:29.539 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 8/37 2024-09-18T00:55:29.551 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:55:29.551 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:55:29.580 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 9/37 2024-09-18T00:55:29.615 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 10/37 2024-09-18T00:55:29.645 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 11/37 2024-09-18T00:55:29.680 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 12/37 2024-09-18T00:55:29.713 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 13/37 2024-09-18T00:55:29.754 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 14/37 2024-09-18T00:55:29.797 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 15/37 2024-09-18T00:55:29.831 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 16/37 2024-09-18T00:55:29.880 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 17/37 2024-09-18T00:55:29.911 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-packaging-20.9-5.el9.noarch 18/37 2024-09-18T00:55:29.985 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 19/37 2024-09-18T00:55:30.015 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:55:30.015 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:55:30.133 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 20/37 2024-09-18T00:55:30.212 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 21/37 2024-09-18T00:55:30.273 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/37 2024-09-18T00:55:30.311 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/37 2024-09-18T00:55:30.360 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/37 2024-09-18T00:55:30.416 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/37 2024-09-18T00:55:30.603 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/37 2024-09-18T00:55:30.657 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 27/37 2024-09-18T00:55:30.735 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/37 2024-09-18T00:55:30.824 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 29/37 2024-09-18T00:55:30.873 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/37 2024-09-18T00:55:30.993 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/37 2024-09-18T00:55:31.164 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-modules-core-2:19.1.1-330.gc2ddf972.el9.n 32/37 2024-09-18T00:55:31.192 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-2:19.1.1-330.gc2ddf972.el9.x86_64 33/37 2024-09-18T00:55:31.218 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-2:19.1.1-330.gc2ddf972.el9.x86_64 33/37 2024-09-18T00:55:31.218 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:31.218 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-18T00:55:31.218 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-18T00:55:31.218 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-18T00:55:31.218 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:31.404 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:55:31.473 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/37 2024-09-18T00:55:31.523 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/37 2024-09-18T00:55:31.550 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/37 2024-09-18T00:55:31.620 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 4/37 2024-09-18T00:55:31.666 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : luarocks-3.9.2-5.el9.noarch 5/37 2024-09-18T00:55:31.698 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 6/37 2024-09-18T00:55:31.790 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 7/37 2024-09-18T00:55:31.868 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 8/37 2024-09-18T00:55:31.909 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 9/37 2024-09-18T00:55:31.945 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 10/37 2024-09-18T00:55:31.974 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 11/37 2024-09-18T00:55:32.009 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 12/37 2024-09-18T00:55:32.042 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 13/37 2024-09-18T00:55:32.083 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 14/37 2024-09-18T00:55:32.118 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 15/37 2024-09-18T00:55:32.152 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 16/37 2024-09-18T00:55:32.201 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 17/37 2024-09-18T00:55:32.233 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-packaging-20.9-5.el9.noarch 18/37 2024-09-18T00:55:32.305 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 19/37 2024-09-18T00:55:32.444 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 20/37 2024-09-18T00:55:32.532 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 21/37 2024-09-18T00:55:32.585 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/37 2024-09-18T00:55:32.624 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/37 2024-09-18T00:55:32.659 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-osd-2:19.1.1-330.gc2ddf972.el9.x86_64 34/37 2024-09-18T00:55:32.673 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/37 2024-09-18T00:55:32.686 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-osd-2:19.1.1-330.gc2ddf972.el9.x86_64 34/37 2024-09-18T00:55:32.687 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:32.687 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-18T00:55:32.687 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-18T00:55:32.687 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-18T00:55:32.687 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:32.728 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/37 2024-09-18T00:55:32.913 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/37 2024-09-18T00:55:32.970 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 27/37 2024-09-18T00:55:33.048 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/37 2024-09-18T00:55:33.126 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mon-2:19.1.1-330.gc2ddf972.el9.x86_64 35/37 2024-09-18T00:55:33.136 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 29/37 2024-09-18T00:55:33.150 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mon-2:19.1.1-330.gc2ddf972.el9.x86_64 35/37 2024-09-18T00:55:33.150 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:33.150 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-18T00:55:33.150 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-18T00:55:33.151 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-18T00:55:33.151 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:33.186 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/37 2024-09-18T00:55:33.301 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/37 2024-09-18T00:55:33.385 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mds-2:19.1.1-330.gc2ddf972.el9.x86_64 36/37 2024-09-18T00:55:33.409 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mds-2:19.1.1-330.gc2ddf972.el9.x86_64 36/37 2024-09-18T00:55:33.409 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:33.409 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-18T00:55:33.409 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-18T00:55:33.410 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-18T00:55:33.410 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:33.466 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mgr-modules-core-2:19.1.1-330.gc2ddf972.el9.n 32/37 2024-09-18T00:55:33.474 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-2:19.1.1-330.gc2ddf972.el9.x86_64 37/37 2024-09-18T00:55:33.488 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mgr-2:19.1.1-330.gc2ddf972.el9.x86_64 33/37 2024-09-18T00:55:33.515 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mgr-2:19.1.1-330.gc2ddf972.el9.x86_64 33/37 2024-09-18T00:55:33.515 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:33.515 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-18T00:55:33.515 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-18T00:55:33.515 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-18T00:55:33.516 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:34.947 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-osd-2:19.1.1-330.gc2ddf972.el9.x86_64 34/37 2024-09-18T00:55:34.975 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-osd-2:19.1.1-330.gc2ddf972.el9.x86_64 34/37 2024-09-18T00:55:34.975 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:34.975 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-18T00:55:34.975 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-18T00:55:34.976 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-18T00:55:34.976 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:35.259 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-2:19.1.1-330.gc2ddf972.el9.x86_64 37/37 2024-09-18T00:55:35.259 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-2:19.1.1-330.gc2ddf972.el9.x86_64 1/37 2024-09-18T00:55:35.259 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mds-2:19.1.1-330.gc2ddf972.el9.x86_64 2/37 2024-09-18T00:55:35.259 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-2:19.1.1-330.gc2ddf972.el9.x86_64 3/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mon-2:19.1.1-330.gc2ddf972.el9.x86_64 4/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-osd-2:19.1.1-330.gc2ddf972.el9.x86_64 5/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-modules-core-2:19.1.1-330.gc2ddf972.el9.n 6/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : lua-5.4.4-4.el9.x86_64 13/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 14/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 15/37 2024-09-18T00:55:35.260 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 16/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 17/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 18/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 19/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 20/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 21/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 22/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 23/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 24/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 25/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 26/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 27/37 2024-09-18T00:55:35.261 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 28/37 2024-09-18T00:55:35.262 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 29/37 2024-09-18T00:55:35.262 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 30/37 2024-09-18T00:55:35.262 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 31/37 2024-09-18T00:55:35.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 32/37 2024-09-18T00:55:35.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 33/37 2024-09-18T00:55:35.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 34/37 2024-09-18T00:55:35.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 35/37 2024-09-18T00:55:35.263 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 36/37 2024-09-18T00:55:35.406 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mon-2:19.1.1-330.gc2ddf972.el9.x86_64 35/37 2024-09-18T00:55:35.430 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mon-2:19.1.1-330.gc2ddf972.el9.x86_64 35/37 2024-09-18T00:55:35.431 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:35.431 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-18T00:55:35.431 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-18T00:55:35.431 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-18T00:55:35.431 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:35.665 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mds-2:19.1.1-330.gc2ddf972.el9.x86_64 36/37 2024-09-18T00:55:35.689 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mds-2:19.1.1-330.gc2ddf972.el9.x86_64 36/37 2024-09-18T00:55:35.689 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:35.689 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-18T00:55:35.689 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-18T00:55:35.689 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-18T00:55:35.689 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:35.753 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-2:19.1.1-330.gc2ddf972.el9.x86_64 37/37 2024-09-18T00:55:36.379 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 37/37 2024-09-18T00:55:36.379 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:36.379 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:55:36.379 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:36.379 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mds-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:36.379 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:36.379 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-modules-core-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:55:36.380 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mon-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:36.380 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-osd-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:36.380 INFO:teuthology.orchestra.run.smithi150.stdout: lua-5.4.4-4.el9.x86_64 2024-09-18T00:55:36.380 INFO:teuthology.orchestra.run.smithi150.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-09-18T00:55:36.380 INFO:teuthology.orchestra.run.smithi150.stdout: luarocks-3.9.2-5.el9.noarch 2024-09-18T00:55:36.380 INFO:teuthology.orchestra.run.smithi150.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-18T00:55:36.380 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-18T00:55:36.380 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-18T00:55:36.380 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-18T00:55:36.380 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ply-3.11-14.el9.noarch 2024-09-18T00:55:36.381 INFO:teuthology.orchestra.run.smithi150.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:36.382 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:55:36.756 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-base 2024-09-18T00:55:37.342 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:52 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:55:37.430 INFO:teuthology.orchestra.run.smithi150.stdout:Package ceph-base-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:55:37.461 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-2:19.1.1-330.gc2ddf972.el9.x86_64 37/37 2024-09-18T00:55:37.461 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-2:19.1.1-330.gc2ddf972.el9.x86_64 1/37 2024-09-18T00:55:37.461 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mds-2:19.1.1-330.gc2ddf972.el9.x86_64 2/37 2024-09-18T00:55:37.461 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-2:19.1.1-330.gc2ddf972.el9.x86_64 3/37 2024-09-18T00:55:37.461 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mon-2:19.1.1-330.gc2ddf972.el9.x86_64 4/37 2024-09-18T00:55:37.461 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-osd-2:19.1.1-330.gc2ddf972.el9.x86_64 5/37 2024-09-18T00:55:37.461 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-modules-core-2:19.1.1-330.gc2ddf972.el9.n 6/37 2024-09-18T00:55:37.461 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/37 2024-09-18T00:55:37.461 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/37 2024-09-18T00:55:37.461 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/37 2024-09-18T00:55:37.461 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : lua-5.4.4-4.el9.x86_64 13/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 14/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 15/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 16/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 17/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 18/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 19/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 20/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 21/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 22/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 23/37 2024-09-18T00:55:37.462 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 24/37 2024-09-18T00:55:37.463 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 25/37 2024-09-18T00:55:37.464 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 26/37 2024-09-18T00:55:37.464 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 27/37 2024-09-18T00:55:37.464 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 28/37 2024-09-18T00:55:37.464 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 29/37 2024-09-18T00:55:37.464 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 30/37 2024-09-18T00:55:37.464 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 31/37 2024-09-18T00:55:37.464 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 32/37 2024-09-18T00:55:37.464 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 33/37 2024-09-18T00:55:37.464 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 34/37 2024-09-18T00:55:37.464 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 35/37 2024-09-18T00:55:37.464 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 36/37 2024-09-18T00:55:37.472 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:55:37.473 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-09-18T00:55:37.473 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:55:37.537 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install cephadm 2024-09-18T00:55:38.115 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:53 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:55:38.245 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:55:38.246 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-09-18T00:55:38.246 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:38.246 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:55:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: cephadm noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 765 k 2024-09-18T00:55:38.246 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:38.246 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:55:38.246 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:38.246 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-09-18T00:55:38.247 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:38.247 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 765 k 2024-09-18T00:55:38.247 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 771 k 2024-09-18T00:55:38.247 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:55:38.426 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 37/37 2024-09-18T00:55:38.426 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:38.426 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:55:38.426 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:38.426 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mds-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:38.426 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:38.426 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-modules-core-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:55:38.426 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mon-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:38.426 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-osd-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:38.426 INFO:teuthology.orchestra.run.smithi078.stdout: lua-5.4.4-4.el9.x86_64 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: luarocks-3.9.2-5.el9.noarch 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-18T00:55:38.427 INFO:teuthology.orchestra.run.smithi078.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ply-3.11-14.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-18T00:55:38.428 INFO:teuthology.orchestra.run.smithi078.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-18T00:55:38.429 INFO:teuthology.orchestra.run.smithi078.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-18T00:55:38.429 INFO:teuthology.orchestra.run.smithi078.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-18T00:55:38.429 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:38.429 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:55:38.526 INFO:teuthology.orchestra.run.smithi150.stdout:cephadm-19.1.1-330.gc2ddf972.el9.noarch.rpm 2.7 MB/s | 765 kB 00:00 2024-09-18T00:55:38.527 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:38.527 INFO:teuthology.orchestra.run.smithi150.stdout:Total 2.7 MB/s | 765 kB 00:00 2024-09-18T00:55:38.527 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:55:38.532 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:55:38.532 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:55:38.538 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:55:38.538 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:55:38.608 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:55:38.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-base 2024-09-18T00:55:39.103 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:55:39.197 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:55:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:53 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:55:39.443 INFO:teuthology.orchestra.run.smithi078.stdout:Package ceph-base-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:55:39.485 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:55:39.486 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-18T00:55:39.486 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:55:39.548 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install cephadm 2024-09-18T00:55:39.569 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:55:39.826 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:55:39.826 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:39.827 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:55:39.827 INFO:teuthology.orchestra.run.smithi150.stdout: cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:55:39.827 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:39.827 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:55:40.066 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-immutable-object-cache 2024-09-18T00:55:40.126 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:54 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:55:40.257 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:55:40.258 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:40.258 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-18T00:55:40.258 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:40.258 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:55:40.258 INFO:teuthology.orchestra.run.smithi078.stdout: cephadm noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 765 k 2024-09-18T00:55:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:40.259 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:55:40.259 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:40.259 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-18T00:55:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:40.259 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 765 k 2024-09-18T00:55:40.259 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 771 k 2024-09-18T00:55:40.259 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:55:40.514 INFO:teuthology.orchestra.run.smithi078.stdout:cephadm-19.1.1-330.gc2ddf972.el9.noarch.rpm 2.9 MB/s | 765 kB 00:00 2024-09-18T00:55:40.515 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:40.515 INFO:teuthology.orchestra.run.smithi078.stdout:Total 2.9 MB/s | 765 kB 00:00 2024-09-18T00:55:40.515 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:55:40.520 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:55:40.520 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:55:40.526 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:55:40.526 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:55:40.596 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:55:40.656 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:55 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:55:40.791 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:55:40.792 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:40.792 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2024-09-18T00:55:40.792 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:40.792 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:55:40.792 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-immutable-object-cache x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 145 k 2024-09-18T00:55:40.792 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:40.792 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:55:40.792 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:40.792 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-09-18T00:55:40.792 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:40.793 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 145 k 2024-09-18T00:55:40.793 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 435 k 2024-09-18T00:55:40.793 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:55:41.017 INFO:teuthology.orchestra.run.smithi150.stdout:ceph-immutable-object-cache-19.1.1-330.gc2ddf97 648 kB/s | 145 kB 00:00 2024-09-18T00:55:41.017 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:41.018 INFO:teuthology.orchestra.run.smithi150.stdout:Total 644 kB/s | 145 kB 00:00 2024-09-18T00:55:41.018 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:55:41.025 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:55:41.026 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:55:41.082 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:55:41.083 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:55:41.092 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:55:41.185 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:55:41.254 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:55:41.327 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-immutable-object-cache-2:19.1.1-330.gc2ddf972.e 1/1 2024-09-18T00:55:41.351 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.1.1-330.gc2ddf972.e 1/1 2024-09-18T00:55:41.351 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:41.351 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-18T00:55:41.351 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:41.570 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:55:41.829 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:55:41.829 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:41.829 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:55:41.829 INFO:teuthology.orchestra.run.smithi078.stdout: cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:55:41.829 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:41.829 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:55:42.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-immutable-object-cache 2024-09-18T00:55:42.266 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-immutable-object-cache-2:19.1.1-330.gc2ddf972.e 1/1 2024-09-18T00:55:42.267 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:42.267 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:55:42.267 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-immutable-object-cache-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:42.267 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:42.267 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:55:42.512 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr 2024-09-18T00:55:42.629 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:56 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:55:42.761 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:55:42.761 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:42.762 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repo Size 2024-09-18T00:55:42.762 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:42.762 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:55:42.762 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-immutable-object-cache x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 145 k 2024-09-18T00:55:42.762 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:42.762 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:55:42.762 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:42.762 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-18T00:55:42.762 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:42.763 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 145 k 2024-09-18T00:55:42.763 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 435 k 2024-09-18T00:55:42.763 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:55:42.947 INFO:teuthology.orchestra.run.smithi078.stdout:ceph-immutable-object-cache-19.1.1-330.gc2ddf97 791 kB/s | 145 kB 00:00 2024-09-18T00:55:42.947 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:42.948 INFO:teuthology.orchestra.run.smithi078.stdout:Total 784 kB/s | 145 kB 00:00 2024-09-18T00:55:42.948 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:55:42.955 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:55:42.955 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:55:43.012 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:55:43.012 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:55:43.103 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:58 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:55:43.163 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:55:43.194 INFO:teuthology.orchestra.run.smithi150.stdout:Package ceph-mgr-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:55:43.223 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-immutable-object-cache-2:19.1.1-330.gc2ddf972.e 1/1 2024-09-18T00:55:43.236 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:55:43.237 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-09-18T00:55:43.237 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:55:43.247 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.1.1-330.gc2ddf972.e 1/1 2024-09-18T00:55:43.247 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:55:43.247 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-18T00:55:43.247 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:43.304 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-dashboard 2024-09-18T00:55:43.888 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:58 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:55:44.020 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:55:44.021 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:44.021 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-dashboard noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 3.6 M 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-grafana-dashboards noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 32 k 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-prometheus-alerts noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 16 k 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-09-18T00:55:44.022 INFO:teuthology.orchestra.run.smithi150.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-09-18T00:55:44.023 INFO:teuthology.orchestra.run.smithi150.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-18T00:55:44.023 INFO:teuthology.orchestra.run.smithi150.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-18T00:55:44.023 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:44.023 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:55:44.023 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:55:44.023 INFO:teuthology.orchestra.run.smithi150.stdout:Install 12 Packages 2024-09-18T00:55:44.023 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:44.023 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 8.7 M 2024-09-18T00:55:44.024 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 104 M 2024-09-18T00:55:44.024 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:55:44.211 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-immutable-object-cache-2:19.1.1-330.gc2ddf972.e 1/1 2024-09-18T00:55:44.212 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:44.212 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:55:44.212 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-immutable-object-cache-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:55:44.212 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:44.212 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:55:44.404 INFO:teuthology.orchestra.run.smithi150.stdout:(1/12): ceph-prometheus-alerts-19.1.1-330.gc2dd 97 kB/s | 16 kB 00:00 2024-09-18T00:55:44.429 INFO:teuthology.orchestra.run.smithi150.stdout:(2/12): ceph-grafana-dashboards-19.1.1-330.gc2d 164 kB/s | 32 kB 00:00 2024-09-18T00:55:44.441 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-mgr 2024-09-18T00:55:44.571 INFO:teuthology.orchestra.run.smithi150.stdout:(3/12): ceph-mgr-dashboard-19.1.1-330.gc2ddf972 11 MB/s | 3.6 MB 00:00 2024-09-18T00:55:44.596 INFO:teuthology.orchestra.run.smithi150.stdout:(4/12): python3-protobuf-3.14.0-13.el9.noarch.r 1.6 MB/s | 269 kB 00:00 2024-09-18T00:55:44.655 INFO:teuthology.orchestra.run.smithi150.stdout:(5/12): abseil-cpp-20211102.0-3.el9.x86_64.rpm 9.2 MB/s | 548 kB 00:00 2024-09-18T00:55:44.680 INFO:teuthology.orchestra.run.smithi150.stdout:(6/12): grpc-data-1.46.7-10.el9.noarch.rpm 779 kB/s | 19 kB 00:00 2024-09-18T00:55:44.706 INFO:teuthology.orchestra.run.smithi150.stdout:(7/12): protobuf-3.14.0-13.el9.x86_64.rpm 3.3 MB/s | 1.0 MB 00:00 2024-09-18T00:55:44.773 INFO:teuthology.orchestra.run.smithi150.stdout:(8/12): python3-grpcio-tools-1.46.7-10.el9.x86_ 2.1 MB/s | 144 kB 00:00 2024-09-18T00:55:44.806 INFO:teuthology.orchestra.run.smithi150.stdout:(9/12): protobuf-compiler-3.14.0-13.el9.x86_64. 3.6 MB/s | 863 kB 00:00 2024-09-18T00:55:44.831 INFO:teuthology.orchestra.run.smithi150.stdout:(10/12): python3-repoze-lru-0.7-16.el9.noarch.r 523 kB/s | 31 kB 00:00 2024-09-18T00:55:44.898 INFO:teuthology.orchestra.run.smithi150.stdout:(11/12): python3-routes-2.5.1-5.el9.noarch.rpm 2.0 MB/s | 188 kB 00:00 2024-09-18T00:55:44.940 INFO:teuthology.orchestra.run.smithi150.stdout:(12/12): python3-grpcio-1.46.7-10.el9.x86_64.rp 7.9 MB/s | 2.0 MB 00:00 2024-09-18T00:55:44.943 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:44.944 INFO:teuthology.orchestra.run.smithi150.stdout:Total 9.4 MB/s | 8.7 MB 00:00 2024-09-18T00:55:44.992 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:55:45.009 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:55:45.009 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:55:45.024 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:59 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:55:45.113 INFO:teuthology.orchestra.run.smithi078.stdout:Package ceph-mgr-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:55:45.155 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:55:45.156 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-18T00:55:45.156 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:55:45.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-mgr-dashboard 2024-09-18T00:55:45.221 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:55:45.221 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:55:45.749 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:55:45.790 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:59 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:55:45.833 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 1/12 2024-09-18T00:55:45.868 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 2/12 2024-09-18T00:55:45.914 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/12 2024-09-18T00:55:45.921 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:55:45.922 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:45.922 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-18T00:55:45.922 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:45.922 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-dashboard noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 3.6 M 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-grafana-dashboards noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 32 k 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-prometheus-alerts noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 16 k 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-18T00:55:45.923 INFO:teuthology.orchestra.run.smithi078.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-18T00:55:45.924 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:45.924 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:55:45.924 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:55:45.924 INFO:teuthology.orchestra.run.smithi078.stdout:Install 12 Packages 2024-09-18T00:55:45.924 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:45.924 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 8.7 M 2024-09-18T00:55:45.924 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 104 M 2024-09-18T00:55:45.925 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:55:45.945 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/12 2024-09-18T00:55:46.051 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 5/12 2024-09-18T00:55:46.180 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 6/12 2024-09-18T00:55:46.265 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 7/12 2024-09-18T00:55:46.308 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 8/12 2024-09-18T00:55:46.344 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 9/12 2024-09-18T00:55:46.384 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-prometheus-alerts-2:19.1.1-330.gc2ddf972.el9. 10/12 2024-09-18T00:55:46.558 INFO:teuthology.orchestra.run.smithi078.stdout:(1/12): ceph-grafana-dashboards-19.1.1-330.gc2d 204 kB/s | 32 kB 00:00 2024-09-18T00:55:46.583 INFO:teuthology.orchestra.run.smithi078.stdout:(2/12): ceph-prometheus-alerts-19.1.1-330.gc2dd 91 kB/s | 16 kB 00:00 2024-09-18T00:55:46.800 INFO:teuthology.orchestra.run.smithi078.stdout:(3/12): ceph-mgr-dashboard-19.1.1-330.gc2ddf972 9.0 MB/s | 3.6 MB 00:00 2024-09-18T00:55:46.825 INFO:teuthology.orchestra.run.smithi078.stdout:(4/12): python3-protobuf-3.14.0-13.el9.noarch.r 1.1 MB/s | 269 kB 00:00 2024-09-18T00:55:46.901 INFO:teuthology.orchestra.run.smithi078.stdout:(5/12): abseil-cpp-20211102.0-3.el9.x86_64.rpm 7.1 MB/s | 548 kB 00:00 2024-09-18T00:55:46.918 INFO:teuthology.orchestra.run.smithi078.stdout:(6/12): grpc-data-1.46.7-10.el9.noarch.rpm 1.1 MB/s | 19 kB 00:00 2024-09-18T00:55:46.960 INFO:teuthology.orchestra.run.smithi078.stdout:(7/12): protobuf-3.14.0-13.el9.x86_64.rpm 2.5 MB/s | 1.0 MB 00:00 2024-09-18T00:55:47.027 INFO:teuthology.orchestra.run.smithi078.stdout:(8/12): python3-grpcio-tools-1.46.7-10.el9.x86_ 2.1 MB/s | 144 kB 00:00 2024-09-18T00:55:47.043 INFO:teuthology.orchestra.run.smithi078.stdout:(9/12): python3-repoze-lru-0.7-16.el9.noarch.rp 1.8 MB/s | 31 kB 00:00 2024-09-18T00:55:47.102 INFO:teuthology.orchestra.run.smithi078.stdout:(10/12): python3-routes-2.5.1-5.el9.noarch.rpm 3.6 MB/s | 188 kB 00:00 2024-09-18T00:55:47.169 INFO:teuthology.orchestra.run.smithi078.stdout:(11/12): python3-grpcio-1.46.7-10.el9.x86_64.rp 8.1 MB/s | 2.0 MB 00:00 2024-09-18T00:55:47.443 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-grafana-dashboards-2:19.1.1-330.gc2ddf972.el9 11/12 2024-09-18T00:55:47.477 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-dashboard-2:19.1.1-330.gc2ddf972.el9.noar 12/12 2024-09-18T00:55:48.161 INFO:teuthology.orchestra.run.smithi078.stdout:(12/12): protobuf-compiler-3.14.0-13.el9.x86_64 635 kB/s | 863 kB 00:01 2024-09-18T00:55:48.166 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:48.167 INFO:teuthology.orchestra.run.smithi078.stdout:Total 3.9 MB/s | 8.7 MB 00:02 2024-09-18T00:55:48.220 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:55:48.239 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:55:48.239 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:55:48.443 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:55:48.444 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:55:48.831 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.1.1-330.gc2ddf972.el9.noar 12/12 2024-09-18T00:55:48.831 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-grafana-dashboards-2:19.1.1-330.gc2ddf972.el9 1/12 2024-09-18T00:55:48.831 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-dashboard-2:19.1.1-330.gc2ddf972.el9.noar 2/12 2024-09-18T00:55:48.831 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-prometheus-alerts-2:19.1.1-330.gc2ddf972.el9. 3/12 2024-09-18T00:55:48.831 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 4/12 2024-09-18T00:55:48.832 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 5/12 2024-09-18T00:55:48.832 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 6/12 2024-09-18T00:55:48.832 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 7/12 2024-09-18T00:55:48.832 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 8/12 2024-09-18T00:55:48.832 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 9/12 2024-09-18T00:55:48.832 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/12 2024-09-18T00:55:48.832 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 11/12 2024-09-18T00:55:48.972 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:55:49.053 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 1/12 2024-09-18T00:55:49.089 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 2/12 2024-09-18T00:55:49.137 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/12 2024-09-18T00:55:49.149 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 12/12 2024-09-18T00:55:49.149 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:49.149 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-grafana-dashboards-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-dashboard-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-prometheus-alerts-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: grpc-data-1.46.7-10.el9.noarch 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: protobuf-3.14.0-13.el9.x86_64 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:49.150 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:55:49.259 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/12 2024-09-18T00:55:49.393 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 5/12 2024-09-18T00:55:49.402 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-18T00:55:49.525 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 6/12 2024-09-18T00:55:49.618 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 7/12 2024-09-18T00:55:49.662 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 8/12 2024-09-18T00:55:49.690 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 9/12 2024-09-18T00:55:49.713 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-prometheus-alerts-2:19.1.1-330.gc2ddf972.el9. 10/12 2024-09-18T00:55:49.985 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:04 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:55:50.116 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:55:50.117 INFO:teuthology.orchestra.run.smithi150.stdout:====================================================================================== 2024-09-18T00:55:50.117 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-09-18T00:55:50.117 INFO:teuthology.orchestra.run.smithi150.stdout:====================================================================================== 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-diskprediction-local noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 7.4 M 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-18T00:55:50.118 INFO:teuthology.orchestra.run.smithi150.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-18T00:55:50.119 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:50.119 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:55:50.119 INFO:teuthology.orchestra.run.smithi150.stdout:====================================================================================== 2024-09-18T00:55:50.119 INFO:teuthology.orchestra.run.smithi150.stdout:Install 12 Packages 2024-09-18T00:55:50.119 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:55:50.119 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 38 M 2024-09-18T00:55:50.119 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 212 M 2024-09-18T00:55:50.120 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:55:50.762 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-grafana-dashboards-2:19.1.1-330.gc2ddf972.el9 11/12 2024-09-18T00:55:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mgr-dashboard-2:19.1.1-330.gc2ddf972.el9.noar 12/12 2024-09-18T00:55:50.989 INFO:teuthology.orchestra.run.smithi150.stdout:(1/12): ceph-mgr-diskprediction-local-19.1.1-33 13 MB/s | 7.4 MB 00:00 2024-09-18T00:55:51.239 INFO:teuthology.orchestra.run.smithi150.stdout:(2/12): libquadmath-11.5.0-2.el9.x86_64.rpm 237 kB/s | 191 kB 00:00 2024-09-18T00:55:51.273 INFO:teuthology.orchestra.run.smithi150.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 115 kB/s | 33 kB 00:00 2024-09-18T00:55:51.381 INFO:teuthology.orchestra.run.smithi150.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 168 kB/s | 18 kB 00:00 2024-09-18T00:55:51.423 INFO:teuthology.orchestra.run.smithi150.stdout:(5/12): libgfortran-11.5.0-2.el9.x86_64.rpm 808 kB/s | 801 kB 00:00 2024-09-18T00:55:51.490 INFO:teuthology.orchestra.run.smithi150.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 355 kB/s | 38 kB 00:00 2024-09-18T00:55:51.774 INFO:teuthology.orchestra.run.smithi150.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 864 kB/s | 245 kB 00:00 2024-09-18T00:55:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.1.1-330.gc2ddf972.el9.noar 12/12 2024-09-18T00:55:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-grafana-dashboards-2:19.1.1-330.gc2ddf972.el9 1/12 2024-09-18T00:55:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-dashboard-2:19.1.1-330.gc2ddf972.el9.noar 2/12 2024-09-18T00:55:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-prometheus-alerts-2:19.1.1-330.gc2ddf972.el9. 3/12 2024-09-18T00:55:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 4/12 2024-09-18T00:55:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 5/12 2024-09-18T00:55:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 6/12 2024-09-18T00:55:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 7/12 2024-09-18T00:55:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 8/12 2024-09-18T00:55:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 9/12 2024-09-18T00:55:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/12 2024-09-18T00:55:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 11/12 2024-09-18T00:55:52.349 INFO:teuthology.orchestra.run.smithi150.stdout:(8/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 8.9 MB/s | 5.1 MB 00:00 2024-09-18T00:55:52.360 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 12/12 2024-09-18T00:55:52.360 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:52.360 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:55:52.360 INFO:teuthology.orchestra.run.smithi078.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-09-18T00:55:52.360 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-grafana-dashboards-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:55:52.360 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-dashboard-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:55:52.360 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-prometheus-alerts-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:55:52.360 INFO:teuthology.orchestra.run.smithi078.stdout: grpc-data-1.46.7-10.el9.noarch 2024-09-18T00:55:52.360 INFO:teuthology.orchestra.run.smithi078.stdout: protobuf-3.14.0-13.el9.x86_64 2024-09-18T00:55:52.360 INFO:teuthology.orchestra.run.smithi078.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-09-18T00:55:52.360 INFO:teuthology.orchestra.run.smithi078.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-09-18T00:55:52.361 INFO:teuthology.orchestra.run.smithi078.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-09-18T00:55:52.361 INFO:teuthology.orchestra.run.smithi078.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-09-18T00:55:52.361 INFO:teuthology.orchestra.run.smithi078.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-18T00:55:52.361 INFO:teuthology.orchestra.run.smithi078.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-18T00:55:52.361 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:52.361 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:55:52.408 INFO:teuthology.orchestra.run.smithi150.stdout:(9/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.6 MB/s | 3.0 MB 00:01 2024-09-18T00:55:52.596 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-18T00:55:52.742 INFO:teuthology.orchestra.run.smithi150.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 640 kB/s | 250 kB 00:00 2024-09-18T00:55:53.172 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:07 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:55:53.304 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:55:53.305 INFO:teuthology.orchestra.run.smithi078.stdout:====================================================================================== 2024-09-18T00:55:53.305 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-18T00:55:53.305 INFO:teuthology.orchestra.run.smithi078.stdout:====================================================================================== 2024-09-18T00:55:53.305 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:55:53.305 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-diskprediction-local noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 7.4 M 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-18T00:55:53.306 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:53.307 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:55:53.307 INFO:teuthology.orchestra.run.smithi078.stdout:====================================================================================== 2024-09-18T00:55:53.307 INFO:teuthology.orchestra.run.smithi078.stdout:Install 12 Packages 2024-09-18T00:55:53.307 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:55:53.307 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 38 M 2024-09-18T00:55:53.307 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 212 M 2024-09-18T00:55:53.307 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:55:53.594 INFO:teuthology.orchestra.run.smithi078.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 874 kB/s | 191 kB 00:00 2024-09-18T00:55:53.644 INFO:teuthology.orchestra.run.smithi078.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.9 MB/s | 801 kB 00:00 2024-09-18T00:55:53.669 INFO:teuthology.orchestra.run.smithi078.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 434 kB/s | 33 kB 00:00 2024-09-18T00:55:53.719 INFO:teuthology.orchestra.run.smithi078.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 365 kB/s | 18 kB 00:00 2024-09-18T00:55:53.791 INFO:teuthology.orchestra.run.smithi150.stdout:(11/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 2.1 MB/s | 4.9 MB 00:02 2024-09-18T00:55:53.836 INFO:teuthology.orchestra.run.smithi078.stdout:(5/12): ceph-mgr-diskprediction-local-19.1.1-33 16 MB/s | 7.4 MB 00:00 2024-09-18T00:55:53.853 INFO:teuthology.orchestra.run.smithi078.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 288 kB/s | 38 kB 00:00 2024-09-18T00:55:53.970 INFO:teuthology.orchestra.run.smithi078.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 2.0 MB/s | 245 kB 00:00 2024-09-18T00:55:54.770 INFO:teuthology.orchestra.run.smithi078.stdout:(8/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.7 MB/s | 3.0 MB 00:01 2024-09-18T00:55:54.912 INFO:teuthology.orchestra.run.smithi078.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 1.7 MB/s | 250 kB 00:00 2024-09-18T00:55:55.546 INFO:teuthology.orchestra.run.smithi078.stdout:(10/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 2.9 MB/s | 4.9 MB 00:01 2024-09-18T00:55:55.805 INFO:teuthology.orchestra.run.smithi078.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 2.8 MB/s | 5.1 MB 00:01 2024-09-18T00:55:56.530 INFO:teuthology.orchestra.run.smithi078.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 10 MB/s | 16 MB 00:01 2024-09-18T00:55:56.531 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:56.531 INFO:teuthology.orchestra.run.smithi078.stdout:Total 12 MB/s | 38 MB 00:03 2024-09-18T00:55:56.887 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:55:56.914 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:55:56.914 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:55:57.341 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:55:57.341 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:55:57.733 INFO:teuthology.orchestra.run.smithi150.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 3.1 MB/s | 16 MB 00:05 2024-09-18T00:55:57.734 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:55:57.734 INFO:teuthology.orchestra.run.smithi150.stdout:Total 5.0 MB/s | 38 MB 00:07 2024-09-18T00:55:58.087 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:55:58.114 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:55:58.114 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:55:58.123 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:55:58.199 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-18T00:55:58.236 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-18T00:55:58.335 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-18T00:55:58.379 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-18T00:55:58.547 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:55:58.547 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:55:58.563 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-18T00:55:58.594 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-18T00:55:58.671 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-18T00:55:59.311 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:55:59.319 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-18T00:55:59.388 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-18T00:55:59.425 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-18T00:55:59.432 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-18T00:55:59.523 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-18T00:55:59.574 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-18T00:55:59.752 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-18T00:55:59.781 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-18T00:55:59.858 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-18T00:56:00.518 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-18T00:56:00.629 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-18T00:56:00.671 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-18T00:56:01.713 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-18T00:56:01.735 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mgr-diskprediction-local-2:19.1.1-330.gc2ddf9 12/12 2024-09-18T00:56:01.895 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-18T00:56:02.941 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-18T00:56:02.972 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-diskprediction-local-2:19.1.1-330.gc2ddf9 12/12 2024-09-18T00:56:04.250 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.1.1-330.gc2ddf9 12/12 2024-09-18T00:56:04.250 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.1.1-330.gc2ddf9 1/12 2024-09-18T00:56:04.250 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-18T00:56:04.250 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-18T00:56:04.250 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-18T00:56:04.250 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-18T00:56:04.251 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-18T00:56:04.251 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-18T00:56:04.251 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-18T00:56:04.251 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-18T00:56:04.251 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-18T00:56:04.251 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-18T00:56:04.662 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-18T00:56:04.662 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:04.662 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:56:04.662 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-diskprediction-local-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:56:04.662 INFO:teuthology.orchestra.run.smithi078.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-18T00:56:04.662 INFO:teuthology.orchestra.run.smithi078.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-18T00:56:04.662 INFO:teuthology.orchestra.run.smithi078.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-18T00:56:04.662 INFO:teuthology.orchestra.run.smithi078.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-18T00:56:04.662 INFO:teuthology.orchestra.run.smithi078.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-18T00:56:04.662 INFO:teuthology.orchestra.run.smithi078.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-18T00:56:04.663 INFO:teuthology.orchestra.run.smithi078.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-18T00:56:04.663 INFO:teuthology.orchestra.run.smithi078.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-18T00:56:04.663 INFO:teuthology.orchestra.run.smithi078.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-18T00:56:04.663 INFO:teuthology.orchestra.run.smithi078.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-18T00:56:04.663 INFO:teuthology.orchestra.run.smithi078.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-18T00:56:04.663 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:04.663 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:05.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-mgr-rook 2024-09-18T00:56:05.488 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.1.1-330.gc2ddf9 12/12 2024-09-18T00:56:05.488 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.1.1-330.gc2ddf9 1/12 2024-09-18T00:56:05.488 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-18T00:56:05.488 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-18T00:56:05.488 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-18T00:56:05.488 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-18T00:56:05.488 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-18T00:56:05.488 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-18T00:56:05.488 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-18T00:56:05.488 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-18T00:56:05.488 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-18T00:56:05.489 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-18T00:56:05.633 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:19 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:05.764 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:05.765 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:05.765 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-18T00:56:05.765 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:05.765 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:56:05.765 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-rook noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 49 k 2024-09-18T00:56:05.765 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-18T00:56:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-18T00:56:05.767 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:05.767 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:56:05.767 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:05.767 INFO:teuthology.orchestra.run.smithi078.stdout:Install 13 Packages 2024-09-18T00:56:05.767 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:05.767 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 2.2 M 2024-09-18T00:56:05.768 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 26 M 2024-09-18T00:56:05.768 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:56:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-18T00:56:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:05.917 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:56:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-diskprediction-local-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:56:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-18T00:56:05.917 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-18T00:56:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-18T00:56:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-18T00:56:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-18T00:56:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-18T00:56:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-18T00:56:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-18T00:56:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-18T00:56:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-18T00:56:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-18T00:56:05.918 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:05.918 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:06.244 INFO:teuthology.orchestra.run.smithi078.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 143 kB/s | 19 kB 00:00 2024-09-18T00:56:06.270 INFO:teuthology.orchestra.run.smithi078.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 161 kB/s | 26 kB 00:00 2024-09-18T00:56:06.295 INFO:teuthology.orchestra.run.smithi078.stdout:(3/13): ceph-mgr-rook-19.1.1-330.gc2ddf972.el9. 268 kB/s | 49 kB 00:00 2024-09-18T00:56:06.309 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-rook 2024-09-18T00:56:06.437 INFO:teuthology.orchestra.run.smithi078.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 952 kB/s | 159 kB 00:00 2024-09-18T00:56:06.504 INFO:teuthology.orchestra.run.smithi078.stdout:(5/13): python3-requests-oauthlib-1.3.0-12.el9. 803 kB/s | 54 kB 00:00 2024-09-18T00:56:06.546 INFO:teuthology.orchestra.run.smithi078.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 774 kB/s | 32 kB 00:00 2024-09-18T00:56:06.571 INFO:teuthology.orchestra.run.smithi078.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.0 MB/s | 279 kB 00:00 2024-09-18T00:56:06.596 INFO:teuthology.orchestra.run.smithi078.stdout:(8/13): python3-certifi-2023.05.07-4.el9.noarch 280 kB/s | 14 kB 00:00 2024-09-18T00:56:06.663 INFO:teuthology.orchestra.run.smithi078.stdout:(9/13): python3-google-auth-2.34.0-1.el9.noarch 2.3 MB/s | 220 kB 00:00 2024-09-18T00:56:06.697 INFO:teuthology.orchestra.run.smithi078.stdout:(10/13): python3-kubernetes-26.1.0-2.el9.noarch 11 MB/s | 1.0 MB 00:00 2024-09-18T00:56:06.722 INFO:teuthology.orchestra.run.smithi078.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 1.0 MB/s | 59 kB 00:00 2024-09-18T00:56:06.747 INFO:teuthology.orchestra.run.smithi078.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.8 MB/s | 90 kB 00:00 2024-09-18T00:56:06.892 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:21 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:07.023 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:07.025 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:07.025 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-09-18T00:56:07.025 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:07.025 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:56:07.025 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-rook noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 49 k 2024-09-18T00:56:07.025 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-09-18T00:56:07.025 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-18T00:56:07.025 INFO:teuthology.orchestra.run.smithi150.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:07.026 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:56:07.027 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:07.027 INFO:teuthology.orchestra.run.smithi150.stdout:Install 13 Packages 2024-09-18T00:56:07.027 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:07.027 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 2.2 M 2024-09-18T00:56:07.027 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 26 M 2024-09-18T00:56:07.027 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:56:07.232 INFO:teuthology.orchestra.run.smithi150.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 173 kB/s | 19 kB 00:00 2024-09-18T00:56:07.257 INFO:teuthology.orchestra.run.smithi150.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 189 kB/s | 26 kB 00:00 2024-09-18T00:56:07.281 INFO:teuthology.orchestra.run.smithi078.stdout:(13/13): python3-oauthlib-3.1.1-5.el9.noarch.rp 214 kB/s | 222 kB 00:01 2024-09-18T00:56:07.283 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:07.284 INFO:teuthology.orchestra.run.smithi078.stdout:Total 1.5 MB/s | 2.2 MB 00:01 2024-09-18T00:56:07.315 INFO:teuthology.orchestra.run.smithi150.stdout:(3/13): ceph-mgr-rook-19.1.1-330.gc2ddf972.el9. 254 kB/s | 49 kB 00:00 2024-09-18T00:56:07.357 INFO:teuthology.orchestra.run.smithi150.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.7 MB/s | 222 kB 00:00 2024-09-18T00:56:07.358 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:56:07.375 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:56:07.375 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:56:07.382 INFO:teuthology.orchestra.run.smithi150.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.2 MB/s | 159 kB 00:00 2024-09-18T00:56:07.408 INFO:teuthology.orchestra.run.smithi150.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-09-18T00:56:07.458 INFO:teuthology.orchestra.run.smithi150.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 427 kB/s | 32 kB 00:00 2024-09-18T00:56:07.471 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:56:07.472 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:56:07.483 INFO:teuthology.orchestra.run.smithi150.stdout:(8/13): python3-certifi-2023.05.07-4.el9.noarch 188 kB/s | 14 kB 00:00 2024-09-18T00:56:07.508 INFO:teuthology.orchestra.run.smithi150.stdout:(9/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.4 MB/s | 279 kB 00:00 2024-09-18T00:56:07.550 INFO:teuthology.orchestra.run.smithi150.stdout:(10/13): python3-google-auth-2.34.0-1.el9.noarc 2.3 MB/s | 220 kB 00:00 2024-09-18T00:56:07.584 INFO:teuthology.orchestra.run.smithi150.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 785 kB/s | 59 kB 00:00 2024-09-18T00:56:07.609 INFO:teuthology.orchestra.run.smithi150.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.5 MB/s | 90 kB 00:00 2024-09-18T00:56:07.668 INFO:teuthology.orchestra.run.smithi150.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 5.6 MB/s | 1.0 MB 00:00 2024-09-18T00:56:07.670 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:07.671 INFO:teuthology.orchestra.run.smithi150.stdout:Total 3.4 MB/s | 2.2 MB 00:00 2024-09-18T00:56:07.733 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:56:07.749 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:56:07.749 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:56:07.839 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:56:07.841 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:56:07.990 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:56:08.174 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-18T00:56:08.223 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-18T00:56:08.287 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-18T00:56:08.324 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-18T00:56:08.363 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:56:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-18T00:56:08.468 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-18T00:56:08.538 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-18T00:56:08.568 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-18T00:56:08.593 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-18T00:56:08.617 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-18T00:56:08.649 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-18T00:56:08.711 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-18T00:56:08.753 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-18T00:56:08.854 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-18T00:56:08.955 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-18T00:56:09.004 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-18T00:56:09.335 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-18T00:56:09.381 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-18T00:56:09.415 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-18T00:56:09.463 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-18T00:56:09.485 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mgr-rook-2:19.1.1-330.gc2ddf972.el9.noarch 13/13 2024-09-18T00:56:09.714 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-18T00:56:09.777 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-18T00:56:09.810 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-18T00:56:09.858 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-18T00:56:09.880 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-rook-2:19.1.1-330.gc2ddf972.el9.noarch 13/13 2024-09-18T00:56:10.275 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mgr-rook-2:19.1.1-330.gc2ddf972.el9.noarch 13/13 2024-09-18T00:56:10.275 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-rook-2:19.1.1-330.gc2ddf972.el9.noarch 1/13 2024-09-18T00:56:10.275 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-18T00:56:10.275 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-18T00:56:10.276 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-18T00:56:10.276 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-18T00:56:10.276 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-18T00:56:10.276 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-18T00:56:10.276 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-18T00:56:10.276 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-18T00:56:10.276 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-18T00:56:10.276 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-18T00:56:10.276 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-18T00:56:10.608 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-18T00:56:10.608 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:10.608 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:56:10.608 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-rook-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:56:10.608 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-18T00:56:10.608 INFO:teuthology.orchestra.run.smithi078.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-18T00:56:10.608 INFO:teuthology.orchestra.run.smithi078.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-18T00:56:10.608 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-18T00:56:10.608 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-18T00:56:10.608 INFO:teuthology.orchestra.run.smithi078.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-18T00:56:10.608 INFO:teuthology.orchestra.run.smithi078.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-18T00:56:10.609 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-18T00:56:10.609 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-18T00:56:10.609 INFO:teuthology.orchestra.run.smithi078.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-18T00:56:10.609 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-18T00:56:10.609 INFO:teuthology.orchestra.run.smithi078.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-18T00:56:10.609 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:10.609 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:10.709 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-rook-2:19.1.1-330.gc2ddf972.el9.noarch 13/13 2024-09-18T00:56:10.709 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-rook-2:19.1.1-330.gc2ddf972.el9.noarch 1/13 2024-09-18T00:56:10.709 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-18T00:56:10.709 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-18T00:56:10.710 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-18T00:56:10.710 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-18T00:56:10.710 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-18T00:56:10.710 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-18T00:56:10.710 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-18T00:56:10.710 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-18T00:56:10.710 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-18T00:56:10.710 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-18T00:56:10.710 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-18T00:56:10.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-mgr-cephadm 2024-09-18T00:56:11.047 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-18T00:56:11.047 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:11.047 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:56:11.047 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-rook-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:56:11.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-18T00:56:11.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-18T00:56:11.047 INFO:teuthology.orchestra.run.smithi150.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-18T00:56:11.048 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-18T00:56:11.048 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-18T00:56:11.048 INFO:teuthology.orchestra.run.smithi150.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-18T00:56:11.048 INFO:teuthology.orchestra.run.smithi150.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-18T00:56:11.048 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-18T00:56:11.048 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-18T00:56:11.048 INFO:teuthology.orchestra.run.smithi150.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-18T00:56:11.048 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-18T00:56:11.048 INFO:teuthology.orchestra.run.smithi150.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-18T00:56:11.048 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:11.048 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:11.315 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-cephadm 2024-09-18T00:56:11.452 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:25 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:11.584 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:11.585 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:11.585 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-18T00:56:11.585 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:11.585 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-cephadm noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 146 k 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout:Install 6 Packages 2024-09-18T00:56:11.586 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:11.587 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 7.0 M 2024-09-18T00:56:11.587 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 33 M 2024-09-18T00:56:11.587 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:56:11.896 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:26 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:12.028 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:12.029 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-09-18T00:56:12.029 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:12.029 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:56:12.029 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-cephadm noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 146 k 2024-09-18T00:56:12.029 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-09-18T00:56:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-18T00:56:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-18T00:56:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-18T00:56:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-18T00:56:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-18T00:56:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:12.030 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:56:12.030 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:12.030 INFO:teuthology.orchestra.run.smithi150.stdout:Install 6 Packages 2024-09-18T00:56:12.030 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:12.031 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 7.0 M 2024-09-18T00:56:12.031 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 33 M 2024-09-18T00:56:12.031 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:56:12.395 INFO:teuthology.orchestra.run.smithi150.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.6 MB/s | 248 kB 00:00 2024-09-18T00:56:12.416 INFO:teuthology.orchestra.run.smithi078.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.1 MB/s | 248 kB 00:00 2024-09-18T00:56:12.441 INFO:teuthology.orchestra.run.smithi078.stdout:(2/6): ceph-mgr-cephadm-19.1.1-330.gc2ddf972.el 613 kB/s | 146 kB 00:00 2024-09-18T00:56:12.462 INFO:teuthology.orchestra.run.smithi150.stdout:(2/6): ceph-mgr-cephadm-19.1.1-330.gc2ddf972.el 661 kB/s | 146 kB 00:00 2024-09-18T00:56:12.508 INFO:teuthology.orchestra.run.smithi078.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 865 kB/s | 58 kB 00:00 2024-09-18T00:56:12.520 INFO:teuthology.orchestra.run.smithi150.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 990 kB/s | 58 kB 00:00 2024-09-18T00:56:12.558 INFO:teuthology.orchestra.run.smithi078.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 1.5 MB/s | 74 kB 00:00 2024-09-18T00:56:12.562 INFO:teuthology.orchestra.run.smithi150.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 1.7 MB/s | 74 kB 00:00 2024-09-18T00:56:12.588 INFO:teuthology.orchestra.run.smithi150.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 2.8 MB/s | 548 kB 00:00 2024-09-18T00:56:12.608 INFO:teuthology.orchestra.run.smithi078.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 2.8 MB/s | 548 kB 00:00 2024-09-18T00:56:12.696 INFO:teuthology.orchestra.run.smithi150.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 13 MB/s | 6.0 MB 00:00 2024-09-18T00:56:12.699 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:12.699 INFO:teuthology.orchestra.run.smithi150.stdout:Total 10 MB/s | 7.0 MB 00:00 2024-09-18T00:56:12.769 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:56:12.781 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:56:12.781 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:56:12.879 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:56:12.880 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:56:13.182 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:56:13.351 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-18T00:56:13.415 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-18T00:56:13.634 INFO:teuthology.orchestra.run.smithi078.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 4.2 MB/s | 6.0 MB 00:01 2024-09-18T00:56:13.636 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:13.637 INFO:teuthology.orchestra.run.smithi078.stdout:Total 3.4 MB/s | 7.0 MB 00:02 2024-09-18T00:56:13.708 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:56:13.719 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:56:13.719 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:56:13.817 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:56:13.818 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:56:13.945 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-18T00:56:14.026 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-18T00:56:14.098 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-18T00:56:14.127 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 6/6 2024-09-18T00:56:14.127 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:56:14.296 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-18T00:56:14.351 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-18T00:56:14.861 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-18T00:56:14.950 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 6/6 2024-09-18T00:56:14.950 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 1/6 2024-09-18T00:56:14.950 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-18T00:56:14.950 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-18T00:56:14.951 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-18T00:56:14.951 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-18T00:56:14.962 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-18T00:56:15.035 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-18T00:56:15.064 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-mgr-cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 6/6 2024-09-18T00:56:15.245 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-18T00:56:15.245 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:15.245 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:56:15.245 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:56:15.245 INFO:teuthology.orchestra.run.smithi150.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-18T00:56:15.245 INFO:teuthology.orchestra.run.smithi150.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-18T00:56:15.245 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-18T00:56:15.245 INFO:teuthology.orchestra.run.smithi150.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-18T00:56:15.246 INFO:teuthology.orchestra.run.smithi150.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-18T00:56:15.246 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:15.246 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:15.518 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-fuse 2024-09-18T00:56:15.838 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 6/6 2024-09-18T00:56:15.838 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-mgr-cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 1/6 2024-09-18T00:56:15.838 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-18T00:56:15.838 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-18T00:56:15.838 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-18T00:56:15.838 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-18T00:56:16.097 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:31 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:16.115 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-18T00:56:16.115 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:16.115 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:56:16.115 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-cephadm-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:56:16.115 INFO:teuthology.orchestra.run.smithi078.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-18T00:56:16.115 INFO:teuthology.orchestra.run.smithi078.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-18T00:56:16.115 INFO:teuthology.orchestra.run.smithi078.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-18T00:56:16.115 INFO:teuthology.orchestra.run.smithi078.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-18T00:56:16.115 INFO:teuthology.orchestra.run.smithi078.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-18T00:56:16.116 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:16.116 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:16.230 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:16.230 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:16.230 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-09-18T00:56:16.230 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:16.230 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:56:16.231 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-fuse x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 897 k 2024-09-18T00:56:16.231 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-09-18T00:56:16.231 INFO:teuthology.orchestra.run.smithi150.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-18T00:56:16.231 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:16.231 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:56:16.231 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:16.231 INFO:teuthology.orchestra.run.smithi150.stdout:Install 2 Packages 2024-09-18T00:56:16.231 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:16.232 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 977 k 2024-09-18T00:56:16.232 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 2.9 M 2024-09-18T00:56:16.232 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:56:16.372 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-fuse 2024-09-18T00:56:16.623 INFO:teuthology.orchestra.run.smithi150.stdout:(1/2): ceph-fuse-19.1.1-330.gc2ddf972.el9.x86_6 3.4 MB/s | 897 kB 00:00 2024-09-18T00:56:16.748 INFO:teuthology.orchestra.run.smithi150.stdout:(2/2): fuse-2.9.9-16.el9.x86_64.rpm 209 kB/s | 80 kB 00:00 2024-09-18T00:56:16.749 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:16.749 INFO:teuthology.orchestra.run.smithi150.stdout:Total 1.8 MB/s | 977 kB 00:00 2024-09-18T00:56:16.761 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:56:16.782 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:56:16.782 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:56:16.863 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:56:16.863 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:56:16.955 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:30 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:17.049 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:56:17.088 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:17.089 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:17.089 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-18T00:56:17.089 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:17.089 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:56:17.089 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-fuse x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 897 k 2024-09-18T00:56:17.089 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-18T00:56:17.089 INFO:teuthology.orchestra.run.smithi078.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-18T00:56:17.089 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:17.089 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:56:17.090 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:17.090 INFO:teuthology.orchestra.run.smithi078.stdout:Install 2 Packages 2024-09-18T00:56:17.090 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:17.090 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 977 k 2024-09-18T00:56:17.090 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 2.9 M 2024-09-18T00:56:17.090 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:56:17.198 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-18T00:56:17.263 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 2/2 2024-09-18T00:56:17.402 INFO:teuthology.orchestra.run.smithi078.stdout:(1/2): ceph-fuse-19.1.1-330.gc2ddf972.el9.x86_6 3.1 MB/s | 897 kB 00:00 2024-09-18T00:56:17.794 INFO:teuthology.orchestra.run.smithi078.stdout:(2/2): fuse-2.9.9-16.el9.x86_64.rpm 118 kB/s | 80 kB 00:00 2024-09-18T00:56:17.794 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:17.795 INFO:teuthology.orchestra.run.smithi078.stdout:Total 1.4 MB/s | 977 kB 00:00 2024-09-18T00:56:17.807 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:56:17.829 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:56:17.829 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:56:17.910 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:56:17.910 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:56:17.932 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 2/2 2024-09-18T00:56:17.932 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 1/2 2024-09-18T00:56:18.096 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:56:18.187 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-18T00:56:18.187 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:18.187 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:56:18.187 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-18T00:56:18.188 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:18.188 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:18.265 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-18T00:56:18.342 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 2/2 2024-09-18T00:56:18.458 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-volume 2024-09-18T00:56:19.010 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 2/2 2024-09-18T00:56:19.010 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 1/2 2024-09-18T00:56:19.055 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:34 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:19.190 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:19.191 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:19.191 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-09-18T00:56:19.191 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:19.191 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:56:19.191 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-volume noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 285 k 2024-09-18T00:56:19.191 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:19.191 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:56:19.191 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:19.191 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-09-18T00:56:19.192 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:19.192 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 285 k 2024-09-18T00:56:19.192 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 1.3 M 2024-09-18T00:56:19.192 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:56:19.286 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-18T00:56:19.286 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:19.287 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:56:19.287 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-18T00:56:19.287 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:19.287 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:19.452 INFO:teuthology.orchestra.run.smithi150.stdout:ceph-volume-19.1.1-330.gc2ddf972.el9.noarch.rpm 1.1 MB/s | 285 kB 00:00 2024-09-18T00:56:19.453 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:19.453 INFO:teuthology.orchestra.run.smithi150.stdout:Total 1.1 MB/s | 285 kB 00:00 2024-09-18T00:56:19.453 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:56:19.461 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:56:19.461 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:56:19.485 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:56:19.486 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:56:19.555 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install ceph-volume 2024-09-18T00:56:19.701 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:56:19.762 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-volume-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:56:19.779 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-volume-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:56:19.779 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:56:19.779 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-09-18T00:56:19.779 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:20.148 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:34 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:20.285 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:20.286 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:20.286 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repository Size 2024-09-18T00:56:20.286 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:20.286 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:56:20.286 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-volume noarch 2:19.1.1-330.gc2ddf972.el9 ceph-noarch 285 k 2024-09-18T00:56:20.287 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:20.287 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:56:20.287 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:20.287 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-18T00:56:20.287 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:20.287 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 285 k 2024-09-18T00:56:20.287 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 1.3 M 2024-09-18T00:56:20.287 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:56:20.506 INFO:teuthology.orchestra.run.smithi078.stdout:ceph-volume-19.1.1-330.gc2ddf972.el9.noarch.rpm 1.3 MB/s | 285 kB 00:00 2024-09-18T00:56:20.507 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:20.507 INFO:teuthology.orchestra.run.smithi078.stdout:Total 1.3 MB/s | 285 kB 00:00 2024-09-18T00:56:20.507 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:56:20.515 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:56:20.515 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:56:20.540 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:56:20.540 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:56:20.754 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:56:20.786 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-volume-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:56:20.786 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:20.787 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:56:20.787 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-volume-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:56:20.787 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:20.787 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:20.825 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : ceph-volume-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:56:20.841 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: ceph-volume-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:56:20.841 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:56:20.841 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-09-18T00:56:20.841 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:21.066 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install librados-devel 2024-09-18T00:56:21.663 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:36 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:21.798 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:21.799 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:21.799 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2024-09-18T00:56:21.799 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:21.799 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:56:21.799 INFO:teuthology.orchestra.run.smithi150.stdout: librados-devel x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 127 k 2024-09-18T00:56:21.799 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:21.799 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:56:21.799 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:21.799 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-09-18T00:56:21.799 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:21.799 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 127 k 2024-09-18T00:56:21.800 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 456 k 2024-09-18T00:56:21.800 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:56:21.815 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : ceph-volume-2:19.1.1-330.gc2ddf972.el9.noarch 1/1 2024-09-18T00:56:21.815 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:21.815 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:56:21.815 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-volume-2:19.1.1-330.gc2ddf972.el9.noarch 2024-09-18T00:56:21.815 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:21.815 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:22.007 INFO:teuthology.orchestra.run.smithi150.stdout:librados-devel-19.1.1-330.gc2ddf972.el9.x86_64. 611 kB/s | 127 kB 00:00 2024-09-18T00:56:22.007 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:22.008 INFO:teuthology.orchestra.run.smithi150.stdout:Total 607 kB/s | 127 kB 00:00 2024-09-18T00:56:22.008 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:56:22.013 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:56:22.013 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:56:22.078 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:56:22.078 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:56:22.109 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install librados-devel 2024-09-18T00:56:22.251 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:56:22.355 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librados-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:22.707 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:36 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:22.791 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librados-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:22.840 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:22.840 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repo Size 2024-09-18T00:56:22.840 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:22.840 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:56:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: librados-devel x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 127 k 2024-09-18T00:56:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:22.840 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:56:22.840 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:22.841 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-18T00:56:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:22.841 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 127 k 2024-09-18T00:56:22.841 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 456 k 2024-09-18T00:56:22.841 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:56:23.045 INFO:teuthology.orchestra.run.smithi078.stdout:librados-devel-19.1.1-330.gc2ddf972.el9.x86_64. 625 kB/s | 127 kB 00:00 2024-09-18T00:56:23.045 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:23.045 INFO:teuthology.orchestra.run.smithi078.stdout:Total 619 kB/s | 127 kB 00:00 2024-09-18T00:56:23.046 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:56:23.046 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librados-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:23.046 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:23.046 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:56:23.046 INFO:teuthology.orchestra.run.smithi150.stdout: librados-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:56:23.046 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:23.046 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:23.051 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:56:23.051 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:56:23.115 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:56:23.115 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:56:23.261 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:56:23.266 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install libcephfs2 2024-09-18T00:56:23.368 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : librados-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:23.841 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: librados-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:23.866 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:38 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:23.957 INFO:teuthology.orchestra.run.smithi150.stdout:Package libcephfs2-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:24.003 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:24.004 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-09-18T00:56:24.004 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:24.077 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install libcephfs-devel 2024-09-18T00:56:24.097 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : librados-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:24.097 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:24.097 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:56:24.097 INFO:teuthology.orchestra.run.smithi078.stdout: librados-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:56:24.098 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:24.098 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:24.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install libcephfs2 2024-09-18T00:56:24.659 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:39 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:24.791 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:24.791 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:24.792 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2024-09-18T00:56:24.792 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:24.792 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:56:24.792 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs-devel x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 32 k 2024-09-18T00:56:24.792 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:24.792 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:56:24.792 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:24.792 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-09-18T00:56:24.792 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:24.792 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 32 k 2024-09-18T00:56:24.792 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 142 k 2024-09-18T00:56:24.793 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:56:24.921 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:38 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:24.960 INFO:teuthology.orchestra.run.smithi150.stdout:libcephfs-devel-19.1.1-330.gc2ddf972.el9.x86_64 190 kB/s | 32 kB 00:00 2024-09-18T00:56:24.960 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:24.961 INFO:teuthology.orchestra.run.smithi150.stdout:Total 188 kB/s | 32 kB 00:00 2024-09-18T00:56:24.961 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:56:24.964 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:56:24.964 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:56:24.988 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:56:24.988 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:56:25.010 INFO:teuthology.orchestra.run.smithi078.stdout:Package libcephfs2-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:25.053 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:25.055 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-18T00:56:25.055 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:25.079 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:56:25.126 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install libcephfs-devel 2024-09-18T00:56:25.181 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libcephfs-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:25.529 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libcephfs-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:25.709 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:39 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:25.769 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libcephfs-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:25.769 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:25.769 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:56:25.769 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:56:25.769 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:25.770 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:25.840 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:25.841 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:25.841 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repo Size 2024-09-18T00:56:25.841 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:25.841 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:56:25.841 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs-devel x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 32 k 2024-09-18T00:56:25.841 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:25.842 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:56:25.842 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:25.842 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-18T00:56:25.842 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:25.842 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 32 k 2024-09-18T00:56:25.842 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 142 k 2024-09-18T00:56:25.842 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:56:25.949 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install librados2 2024-09-18T00:56:25.990 INFO:teuthology.orchestra.run.smithi078.stdout:libcephfs-devel-19.1.1-330.gc2ddf972.el9.x86_64 214 kB/s | 32 kB 00:00 2024-09-18T00:56:25.990 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:25.990 INFO:teuthology.orchestra.run.smithi078.stdout:Total 213 kB/s | 32 kB 00:00 2024-09-18T00:56:25.991 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:56:25.994 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:56:25.994 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:56:26.017 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:56:26.017 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:56:26.109 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:56:26.220 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : libcephfs-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:26.523 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:41 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:26.572 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: libcephfs-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:26.612 INFO:teuthology.orchestra.run.smithi150.stdout:Package librados2-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:26.655 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:26.656 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-09-18T00:56:26.656 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:26.721 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install librbd1 2024-09-18T00:56:26.814 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : libcephfs-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:26.814 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:26.814 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:56:26.815 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs-devel-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:56:26.815 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:26.815 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:26.978 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install librados2 2024-09-18T00:56:27.307 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:42 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:27.397 INFO:teuthology.orchestra.run.smithi150.stdout:Package librbd1-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:27.442 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:27.443 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-09-18T00:56:27.443 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:27.517 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-rados 2024-09-18T00:56:27.551 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:41 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:27.640 INFO:teuthology.orchestra.run.smithi078.stdout:Package librados2-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:27.683 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:27.684 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-18T00:56:27.684 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:27.748 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install librbd1 2024-09-18T00:56:28.098 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:43 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:28.187 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-rados-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:28.230 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:28.231 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-09-18T00:56:28.231 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:28.296 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-rgw 2024-09-18T00:56:28.325 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:42 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:28.413 INFO:teuthology.orchestra.run.smithi078.stdout:Package librbd1-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:28.456 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:28.457 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-18T00:56:28.457 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:28.523 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install python3-rados 2024-09-18T00:56:28.885 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:43 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:28.975 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-rgw-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:29.018 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:29.019 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-09-18T00:56:29.019 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:29.093 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-cephfs 2024-09-18T00:56:29.104 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:43 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:29.193 INFO:teuthology.orchestra.run.smithi078.stdout:Package python3-rados-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:29.236 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:29.237 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-18T00:56:29.237 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:29.301 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install python3-rgw 2024-09-18T00:56:29.673 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:44 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:29.763 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-cephfs-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:29.806 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:29.807 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-09-18T00:56:29.807 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:29.872 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-rbd 2024-09-18T00:56:29.877 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:43 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:29.966 INFO:teuthology.orchestra.run.smithi078.stdout:Package python3-rgw-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:30.008 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:30.009 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-18T00:56:30.009 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:30.074 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install python3-cephfs 2024-09-18T00:56:30.461 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:45 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:30.551 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-rbd-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:30.594 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:30.595 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-09-18T00:56:30.595 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:30.650 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:44 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:30.667 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install rbd-fuse 2024-09-18T00:56:30.738 INFO:teuthology.orchestra.run.smithi078.stdout:Package python3-cephfs-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:30.781 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:30.782 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-18T00:56:30.782 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:30.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install python3-rbd 2024-09-18T00:56:31.250 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:46 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:31.382 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:31.384 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: Package Architecture Version Repository Size 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-fuse x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 85 k 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 85 k 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 231 k 2024-09-18T00:56:31.385 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:56:31.425 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:45 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:31.514 INFO:teuthology.orchestra.run.smithi078.stdout:Package python3-rbd-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:31.557 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:31.558 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-18T00:56:31.558 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:31.571 INFO:teuthology.orchestra.run.smithi150.stdout:rbd-fuse-19.1.1-330.gc2ddf972.el9.x86_64.rpm 455 kB/s | 85 kB 00:00 2024-09-18T00:56:31.572 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:31.572 INFO:teuthology.orchestra.run.smithi150.stdout:Total 452 kB/s | 85 kB 00:00 2024-09-18T00:56:31.572 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:56:31.579 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:56:31.579 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:56:31.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install rbd-fuse 2024-09-18T00:56:31.644 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:56:31.644 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:56:31.784 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:56:31.885 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : rbd-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:32.236 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:46 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:32.266 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: rbd-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:32.375 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:32.376 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:32.376 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-09-18T00:56:32.376 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:32.376 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:56:32.376 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-fuse x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 85 k 2024-09-18T00:56:32.376 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:32.376 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:56:32.376 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:32.377 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-18T00:56:32.377 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:32.377 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 85 k 2024-09-18T00:56:32.377 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 231 k 2024-09-18T00:56:32.377 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:56:32.568 INFO:teuthology.orchestra.run.smithi078.stdout:rbd-fuse-19.1.1-330.gc2ddf972.el9.x86_64.rpm 448 kB/s | 85 kB 00:00 2024-09-18T00:56:32.569 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:32.569 INFO:teuthology.orchestra.run.smithi078.stdout:Total 444 kB/s | 85 kB 00:00 2024-09-18T00:56:32.569 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:56:32.570 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : rbd-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:32.570 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:32.570 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:56:32.570 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:56:32.570 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:32.570 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:32.576 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:56:32.576 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:56:32.641 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:56:32.642 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:56:32.804 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install rbd-mirror 2024-09-18T00:56:32.811 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:56:32.901 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : rbd-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:33.282 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: rbd-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:33.403 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:48 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:33.525 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:33.525 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:33.525 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:56:33.525 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-fuse-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:56:33.525 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:33.525 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:33.537 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:33.537 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:33.537 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2024-09-18T00:56:33.537 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:33.537 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:56:33.537 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-mirror x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 3.1 M 2024-09-18T00:56:33.537 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:33.538 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:56:33.538 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:33.538 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-09-18T00:56:33.538 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:33.538 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 3.1 M 2024-09-18T00:56:33.538 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 13 M 2024-09-18T00:56:33.538 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:56:33.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install rbd-mirror 2024-09-18T00:56:33.867 INFO:teuthology.orchestra.run.smithi150.stdout:rbd-mirror-19.1.1-330.gc2ddf972.el9.x86_64.rpm 9.4 MB/s | 3.1 MB 00:00 2024-09-18T00:56:33.867 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:33.868 INFO:teuthology.orchestra.run.smithi150.stdout:Total 9.4 MB/s | 3.1 MB 00:00 2024-09-18T00:56:33.868 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:56:33.877 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:56:33.877 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:56:33.964 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:56:33.964 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:56:34.357 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:48 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:34.422 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:56:34.490 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:34.491 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:34.491 INFO:teuthology.orchestra.run.smithi078.stdout: Package Arch Version Repo Size 2024-09-18T00:56:34.491 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:34.491 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:56:34.491 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-mirror x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 3.1 M 2024-09-18T00:56:34.491 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:34.492 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:56:34.492 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:34.492 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-18T00:56:34.492 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:34.492 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 3.1 M 2024-09-18T00:56:34.492 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 13 M 2024-09-18T00:56:34.492 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:56:34.601 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : rbd-mirror-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:34.627 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: rbd-mirror-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:34.627 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:56:34.627 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-18T00:56:34.628 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-18T00:56:34.628 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-18T00:56:34.628 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:34.821 INFO:teuthology.orchestra.run.smithi078.stdout:rbd-mirror-19.1.1-330.gc2ddf972.el9.x86_64.rpm 9.4 MB/s | 3.1 MB 00:00 2024-09-18T00:56:34.822 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:34.822 INFO:teuthology.orchestra.run.smithi078.stdout:Total 9.4 MB/s | 3.1 MB 00:00 2024-09-18T00:56:34.823 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:56:34.832 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:56:34.832 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:56:34.913 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:56:34.914 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:56:35.346 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:56:35.414 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : rbd-mirror-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:35.440 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: rbd-mirror-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:35.440 INFO:teuthology.orchestra.run.smithi078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-18T00:56:35.440 INFO:teuthology.orchestra.run.smithi078.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-18T00:56:35.440 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-18T00:56:35.440 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-18T00:56:35.440 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:35.669 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : rbd-mirror-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:35.669 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:35.669 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:56:35.669 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-mirror-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:56:35.669 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:35.669 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:35.896 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install rbd-nbd 2024-09-18T00:56:36.493 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:51 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:36.523 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-mirror-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:36.523 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:36.523 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:56:36.523 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-mirror-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:56:36.523 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:36.523 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:36.628 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:36.629 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:36.629 INFO:teuthology.orchestra.run.smithi150.stdout: Package Architecture Version Repository Size 2024-09-18T00:56:36.629 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:36.629 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-09-18T00:56:36.629 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-nbd x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 170 k 2024-09-18T00:56:36.629 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:36.629 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-09-18T00:56:36.630 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-09-18T00:56:36.630 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-09-18T00:56:36.630 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:36.630 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 170 k 2024-09-18T00:56:36.630 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 494 k 2024-09-18T00:56:36.630 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-09-18T00:56:36.759 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install rbd-nbd 2024-09-18T00:56:36.837 INFO:teuthology.orchestra.run.smithi150.stdout:rbd-nbd-19.1.1-330.gc2ddf972.el9.x86_64.rpm 825 kB/s | 170 kB 00:00 2024-09-18T00:56:36.837 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:36.838 INFO:teuthology.orchestra.run.smithi150.stdout:Total 819 kB/s | 170 kB 00:00 2024-09-18T00:56:36.838 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-09-18T00:56:36.846 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-09-18T00:56:36.846 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-09-18T00:56:36.909 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-09-18T00:56:36.909 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-09-18T00:56:37.058 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-09-18T00:56:37.159 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : rbd-nbd-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:37.352 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:51 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:37.485 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:37.485 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:37.485 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-09-18T00:56:37.485 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:37.485 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-18T00:56:37.486 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-nbd x86_64 2:19.1.1-330.gc2ddf972.el9 ceph 170 k 2024-09-18T00:56:37.486 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:37.486 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-18T00:56:37.486 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-18T00:56:37.486 INFO:teuthology.orchestra.run.smithi078.stdout:Install 1 Package 2024-09-18T00:56:37.486 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:37.486 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 170 k 2024-09-18T00:56:37.486 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 494 k 2024-09-18T00:56:37.486 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-18T00:56:37.604 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: rbd-nbd-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:37.724 INFO:teuthology.orchestra.run.smithi078.stdout:rbd-nbd-19.1.1-330.gc2ddf972.el9.x86_64.rpm 717 kB/s | 170 kB 00:00 2024-09-18T00:56:37.724 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-18T00:56:37.725 INFO:teuthology.orchestra.run.smithi078.stdout:Total 713 kB/s | 170 kB 00:00 2024-09-18T00:56:37.725 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-18T00:56:37.733 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-18T00:56:37.733 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-18T00:56:37.797 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-18T00:56:37.797 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-18T00:56:37.869 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : rbd-nbd-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:37.869 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:37.869 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-09-18T00:56:37.869 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-nbd-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:56:37.869 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:37.869 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-18T00:56:38.055 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : rbd-nbd-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:38.088 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install rbd-mirror 2024-09-18T00:56:38.500 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: rbd-nbd-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:38.692 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:53 ago on Wed 18 Sep 2024 12:54:45 AM UTC. 2024-09-18T00:56:38.745 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : rbd-nbd-2:19.1.1-330.gc2ddf972.el9.x86_64 1/1 2024-09-18T00:56:38.746 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:38.746 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-18T00:56:38.746 INFO:teuthology.orchestra.run.smithi078.stdout: rbd-nbd-2:19.1.1-330.gc2ddf972.el9.x86_64 2024-09-18T00:56:38.746 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:38.746 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:38.784 INFO:teuthology.orchestra.run.smithi150.stdout:Package rbd-mirror-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:38.830 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-09-18T00:56:38.831 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-09-18T00:56:38.831 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-09-18T00:56:38.904 DEBUG:teuthology.parallel:result is None 2024-09-18T00:56:38.984 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum -y install rbd-mirror 2024-09-18T00:56:39.589 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:01:53 ago on Wed 18 Sep 2024 12:54:46 AM UTC. 2024-09-18T00:56:39.683 INFO:teuthology.orchestra.run.smithi078.stdout:Package rbd-mirror-2:19.1.1-330.gc2ddf972.el9.x86_64 is already installed. 2024-09-18T00:56:39.728 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-18T00:56:39.729 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-09-18T00:56:39.729 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-18T00:56:39.803 DEBUG:teuthology.parallel:result is None 2024-09-18T00:56:39.804 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-18T00:56:39.945 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-18T00:56:39.966 INFO:teuthology.orchestra.run.smithi078.stdout:19.1.1-330.gc2ddf972.el9 2024-09-18T00:56:39.966 INFO:teuthology.packaging:The installed version of ceph is 19.1.1-330.gc2ddf972.el9 2024-09-18T00:56:39.967 INFO:teuthology.task.install:The correct ceph version 19.1.1-330.gc2ddf972 is installed. 2024-09-18T00:56:39.968 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-18T00:56:40.116 DEBUG:teuthology.orchestra.run.smithi150:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-18T00:56:40.142 INFO:teuthology.orchestra.run.smithi150.stdout:19.1.1-330.gc2ddf972.el9 2024-09-18T00:56:40.143 INFO:teuthology.packaging:The installed version of ceph is 19.1.1-330.gc2ddf972.el9 2024-09-18T00:56:40.143 INFO:teuthology.task.install:The correct ceph version 19.1.1-330.gc2ddf972 is installed. 2024-09-18T00:56:40.144 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-18T00:56:40.145 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:40.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-18T00:56:40.170 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:40.170 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-18T00:56:40.212 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-18T00:56:40.213 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:40.213 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/usr/bin/daemon-helper 2024-09-18T00:56:40.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-18T00:56:40.302 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:40.303 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/usr/bin/daemon-helper 2024-09-18T00:56:40.331 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-18T00:56:40.400 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-18T00:56:40.400 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:40.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-18T00:56:40.425 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-18T00:56:40.490 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:40.490 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-18T00:56:40.518 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-18T00:56:40.585 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-18T00:56:40.585 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:40.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/usr/bin/stdin-killer 2024-09-18T00:56:40.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-18T00:56:40.675 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:40.675 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/usr/bin/stdin-killer 2024-09-18T00:56:40.702 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-18T00:56:40.769 INFO:teuthology.run_tasks:Running task ceph... 2024-09-18T00:56:40.863 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-09-18T00:56:40.863 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 777 /var/log/ceph 2024-09-18T00:56:40.866 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 777 /var/log/ceph 2024-09-18T00:56:40.891 INFO:tasks.ceph:Disabling ceph logrotate... 2024-09-18T00:56:40.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-18T00:56:40.931 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-18T00:56:40.956 INFO:tasks.ceph:Creating extra log directories... 2024-09-18T00:56:40.957 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-18T00:56:40.998 DEBUG:teuthology.orchestra.run.smithi150:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-18T00:56:41.036 INFO:tasks.ceph:Creating ceph cluster cluster1... 2024-09-18T00:56:41.037 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-18T00:56:41.037 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686', 'branch': 'squid', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-lz4 policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}', 'email': 'ceph-qa@ceph.com', 'first_in_suite': False, 'job_id': '7909686', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'squid', 'sha1': 'b0d356b98e08f6bc59ef15652f0d0e4298fc83c7'}}, 'owner': 'scheduled_teuthology@teuthology', 'priority': 100, 'repo': 'https://git.ceph.com/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 1983, 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f', 'sleep_before_teardown': 0, 'subset': '79/128', 'suite': 'rbd', 'suite_branch': 'squid', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph_b0d356b98e08f6bc59ef15652f0d0e4298fc83c7/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph.git', 'suite_sha1': 'b0d356b98e08f6bc59ef15652f0d0e4298fc83c7', 'targets': {'smithi078.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHoBQr+C0uE2Le/lt0B2M0DhCOE+f9NQs6lJvI49ArMnn1SGLqMsYEr4xHrV0h9ypsUL6i4u9Z2dZQH3T3Eifkc=', 'smithi150.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAkxbPf3Tcg64mNNL1gjy0wPGGOSlVPApAHiVtGuSX3ZCnVTGXln5Ad+K5u/FeK/vmT65DuVNC0Z+brnbLwhT6k='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-17_21:16:02', 'tube': 'smithi', 'user': 'teuthology', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-09-18T00:56:41.037 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2024-09-18T00:56:41.063 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2024-09-18T00:56:41.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m0777 -- /var/run/ceph 2024-09-18T00:56:41.118 DEBUG:teuthology.orchestra.run.smithi150:> sudo install -d -m0777 -- /var/run/ceph 2024-09-18T00:56:41.144 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:41.144 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/scratch_devs of=/dev/stdout 2024-09-18T00:56:41.196 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-18T00:56:41.197 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_1 2024-09-18T00:56:41.251 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-18T00:56:41.251 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T00:56:41.251 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-18T00:56:41.252 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T00:56:41.252 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T00:56:41.252 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-18 00:56:35.683959239 +0000 2024-09-18T00:56:41.252 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-18 00:54:01.258177505 +0000 2024-09-18T00:56:41.252 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-18 00:54:01.259177490 +0000 2024-09-18T00:56:41.252 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-18 00:54:01.258177505 +0000 2024-09-18T00:56:41.252 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-18T00:56:41.314 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-18T00:56:41.315 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-18T00:56:41.315 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000176844 s, 2.9 MB/s 2024-09-18T00:56:41.317 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-18T00:56:41.372 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_2 2024-09-18T00:56:41.427 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-18T00:56:41.427 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T00:56:41.428 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-18T00:56:41.428 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T00:56:41.428 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T00:56:41.428 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-18 00:56:35.684959224 +0000 2024-09-18T00:56:41.428 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-18 00:54:01.594172499 +0000 2024-09-18T00:56:41.428 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-18 00:54:01.594172499 +0000 2024-09-18T00:56:41.428 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-18 00:54:01.594172499 +0000 2024-09-18T00:56:41.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-18T00:56:41.490 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-18T00:56:41.491 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-18T00:56:41.491 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000187794 s, 2.7 MB/s 2024-09-18T00:56:41.492 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-18T00:56:41.547 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_3 2024-09-18T00:56:41.602 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-18T00:56:41.602 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T00:56:41.602 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-18T00:56:41.602 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T00:56:41.603 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T00:56:41.603 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-18 00:56:35.684959224 +0000 2024-09-18T00:56:41.603 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-18 00:54:01.913167747 +0000 2024-09-18T00:56:41.603 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-18 00:54:01.913167747 +0000 2024-09-18T00:56:41.603 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-18 00:54:01.913167747 +0000 2024-09-18T00:56:41.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-18T00:56:41.666 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-18T00:56:41.666 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-18T00:56:41.666 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000204608 s, 2.5 MB/s 2024-09-18T00:56:41.667 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-18T00:56:41.723 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_4 2024-09-18T00:56:41.779 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-18T00:56:41.779 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T00:56:41.779 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-18T00:56:41.779 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T00:56:41.779 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T00:56:41.779 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-18 00:56:35.684959224 +0000 2024-09-18T00:56:41.779 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-18 00:54:02.234162964 +0000 2024-09-18T00:56:41.779 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-18 00:54:02.234162964 +0000 2024-09-18T00:56:41.779 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-18 00:54:02.234162964 +0000 2024-09-18T00:56:41.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-18T00:56:41.842 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-18T00:56:41.842 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-18T00:56:41.842 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000184226 s, 2.8 MB/s 2024-09-18T00:56:41.843 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-18T00:56:41.899 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-18T00:56:41.900 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi078.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-18T00:56:41.900 INFO:tasks.ceph:Generating config... 2024-09-18T00:56:41.901 INFO:tasks.ceph:[client] debug rbd = 20 2024-09-18T00:56:41.901 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2024-09-18T00:56:41.901 INFO:tasks.ceph:[client] log to stderr = False 2024-09-18T00:56:41.901 INFO:tasks.ceph:[client] rbd default features = 125 2024-09-18T00:56:41.901 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2024-09-18T00:56:41.901 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:41.901 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:41.901 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:41.902 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:41.902 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:41.902 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:41.902 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:41.902 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:41.902 INFO:tasks.ceph:[global] mon client directed command retry = 5 2024-09-18T00:56:41.902 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-18T00:56:41.902 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2024-09-18T00:56:41.902 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-18T00:56:41.902 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-18T00:56:41.902 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-18T00:56:41.903 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-18T00:56:41.903 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-18T00:56:41.903 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-18T00:56:41.903 INFO:tasks.ceph:[osd] bluestore compression algorithm = lz4 2024-09-18T00:56:41.903 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2024-09-18T00:56:41.903 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-18T00:56:41.903 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-18T00:56:41.903 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-18T00:56:41.903 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-18T00:56:41.903 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-18T00:56:41.903 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-18T00:56:41.903 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-18T00:56:41.903 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-18T00:56:41.904 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-18T00:56:41.904 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-18T00:56:41.904 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-18T00:56:41.904 INFO:tasks.ceph:Setting up cluster1.mon.a... 2024-09-18T00:56:41.904 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2024-09-18T00:56:41.986 INFO:teuthology.orchestra.run.smithi078.stdout:creating /etc/ceph/cluster1.keyring 2024-09-18T00:56:41.989 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2024-09-18T00:56:42.031 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-18T00:56:42.095 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '172.21.15.78')] 2024-09-18T00:56:42.096 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.78', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2024-09-18T00:56:42.097 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:42.097 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-18T00:56:42.151 DEBUG:teuthology.orchestra.run.smithi078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.78 --print /home/ubuntu/cephtest/cluster1.monmap 2024-09-18T00:56:42.231 INFO:teuthology.orchestra.run.smithi078.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2024-09-18T00:56:42.231 INFO:teuthology.orchestra.run.smithi078.stdout:monmaptool: generated fsid 539007ae-4341-41e5-b60c-c3d4ca69948f 2024-09-18T00:56:42.231 INFO:teuthology.orchestra.run.smithi078.stdout:setting min_mon_release = quincy 2024-09-18T00:56:42.231 INFO:teuthology.orchestra.run.smithi078.stdout:epoch 0 2024-09-18T00:56:42.231 INFO:teuthology.orchestra.run.smithi078.stdout:fsid 539007ae-4341-41e5-b60c-c3d4ca69948f 2024-09-18T00:56:42.231 INFO:teuthology.orchestra.run.smithi078.stdout:last_changed 2024-09-18T00:56:42.230999+0000 2024-09-18T00:56:42.231 INFO:teuthology.orchestra.run.smithi078.stdout:created 2024-09-18T00:56:42.230999+0000 2024-09-18T00:56:42.231 INFO:teuthology.orchestra.run.smithi078.stdout:min_mon_release 17 (quincy) 2024-09-18T00:56:42.232 INFO:teuthology.orchestra.run.smithi078.stdout:election_strategy: 1 2024-09-18T00:56:42.232 INFO:teuthology.orchestra.run.smithi078.stdout:0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.a 2024-09-18T00:56:42.232 INFO:teuthology.orchestra.run.smithi078.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2024-09-18T00:56:42.236 DEBUG:teuthology.orchestra.run.smithi078:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-18T00:56:42.288 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 539007ae-4341-41e5-b60c-c3d4ca69948f... 2024-09-18T00:56:42.288 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2024-09-18T00:56:42.330 DEBUG:teuthology.orchestra.run.smithi150:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2024-09-18T00:56:42.372 INFO:teuthology.orchestra.run.smithi078.stdout:[global] 2024-09-18T00:56:42.372 INFO:teuthology.orchestra.run.smithi078.stdout: chdir = "" 2024-09-18T00:56:42.372 INFO:teuthology.orchestra.run.smithi078.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-18T00:56:42.372 INFO:teuthology.orchestra.run.smithi078.stdout: auth supported = cephx 2024-09-18T00:56:42.372 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: filestore xattr use omap = true 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: mon clock drift allowed = 1.000 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: osd crush chooseleaf type = 0 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: auth debug = true 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: ms die on old message = true 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: ms die on bug = true 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: mon pg warn max object skew = 0 2024-09-18T00:56:42.373 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: # disable pg_autoscaler by default for new pools 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: osd pool default size = 2 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd allow primary affinity = true 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd allow pg remap = true 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on legacy crush tunables = false 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on crush straw calc version zero = false 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on no sortbitwise = false 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on osd down out interval zero = false 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on too few osds = false 2024-09-18T00:56:42.374 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: mon_allow_pool_size_one = true 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: osd default data pool replay window = 5 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: mon allow pool delete = true 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: mon cluster log file level = debug 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: debug asserts on shutdown = true 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: mon health detail to clog = false 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: mon host = 172.21.15.78 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: mon client directed command retry = 5 2024-09-18T00:56:42.375 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on pool no app = False 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: ms inject socket failures = 5000 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: fsid = 539007ae-4341-41e5-b60c-c3d4ca69948f 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout:[osd] 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: osd journal size = 100 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: osd scrub load threshold = 5.0 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: osd scrub max interval = 600 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: osd mclock profile = high_recovery_ops 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: osd recover clone overlap = true 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: osd recovery max chunk = 1048576 2024-09-18T00:56:42.376 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug shutdown = true 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug op order = true 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug verify stray on activate = true 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug trim objects = true 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: osd open classes on start = true 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug pg log writeout = true 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: osd deep scrub update digest min age = 30 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: osd map max advance = 10 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.377 INFO:teuthology.orchestra.run.smithi078.stdout: journal zero on create = true 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: filestore ondisk finisher threads = 3 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: filestore apply finisher threads = 3 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: bdev debug aio = true 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug misdirected ops = true 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore block size = 96636764160 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore compression algorithm = lz4 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore compression mode = aggressive 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore fsck on mount = True 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: debug bluefs = 1/20 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: debug bluestore = 1/20 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: debug ms = 1 2024-09-18T00:56:42.378 INFO:teuthology.orchestra.run.smithi078.stdout: debug osd = 20 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: debug rocksdb = 4/10 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd backfillfull_ratio = 0.85 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd full ratio = 0.9 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd nearfull ratio = 0.8 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: osd failsafe full ratio = 0.95 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: osd objectstore = bluestore 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout:[mgr] 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: debug ms = 1 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: debug mgr = 20 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: debug mon = 20 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: debug auth = 20 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min pgs per osd = 4 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min bytes per osd = 10 2024-09-18T00:56:42.379 INFO:teuthology.orchestra.run.smithi078.stdout: mgr/telemetry/nag = false 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout:[mon] 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: debug ms = 1 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: debug mon = 20 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: debug paxos = 20 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: debug auth = 20 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: mon data avail warn = 5 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: mon mgr mkfs grace = 240 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min pgs per osd = 4 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd reporter subtree level = osd 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd prime pg temp = true 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min bytes per osd = 10 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.380 INFO:teuthology.orchestra.run.smithi078.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: auth mon ticket ttl = 660 # 11m 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: auth service ticket ttl = 240 # 4m 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: # don't complain about insecure global_id in the test suite 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: # 1m isn't quite enough 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: mon_down_mkfs_grace = 2m 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_filestore_osds = false 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:42.381 INFO:teuthology.orchestra.run.smithi078.stdout:[client] 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: rgw cache enabled = true 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: rgw enable ops log = true 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: rgw enable usage log = true 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: debug rbd = 20 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: debug rbd_mirror = 15 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: log to stderr = False 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: rbd default features = 125 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: rbd mirror image policy type = simple 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout:[mon.a] 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.0] 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:42.382 INFO:teuthology.orchestra.run.smithi078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:42.383 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.1] 2024-09-18T00:56:42.383 INFO:teuthology.orchestra.run.smithi078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:42.383 INFO:teuthology.orchestra.run.smithi078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:42.383 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.2] 2024-09-18T00:56:42.383 INFO:teuthology.orchestra.run.smithi078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:42.383 INFO:teuthology.orchestra.run.smithi078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:42.383 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.3] 2024-09-18T00:56:42.383 INFO:teuthology.orchestra.run.smithi078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:42.383 INFO:teuthology.orchestra.run.smithi078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:42.384 INFO:teuthology.orchestra.run.smithi150.stdout:[global] 2024-09-18T00:56:42.384 INFO:teuthology.orchestra.run.smithi150.stdout: chdir = "" 2024-09-18T00:56:42.384 INFO:teuthology.orchestra.run.smithi150.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-18T00:56:42.384 INFO:teuthology.orchestra.run.smithi150.stdout: auth supported = cephx 2024-09-18T00:56:42.384 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.384 INFO:teuthology.orchestra.run.smithi150.stdout: filestore xattr use omap = true 2024-09-18T00:56:42.384 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.384 INFO:teuthology.orchestra.run.smithi150.stdout: mon clock drift allowed = 1.000 2024-09-18T00:56:42.384 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.384 INFO:teuthology.orchestra.run.smithi150.stdout: osd crush chooseleaf type = 0 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: auth debug = true 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: ms die on old message = true 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: ms die on bug = true 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: mon pg warn max object skew = 0 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: # disable pg_autoscaler by default for new pools 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: osd pool default size = 2 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.385 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd allow primary affinity = true 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd allow pg remap = true 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on legacy crush tunables = false 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on crush straw calc version zero = false 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on no sortbitwise = false 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on osd down out interval zero = false 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on too few osds = false 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: mon_allow_pool_size_one = true 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.386 INFO:teuthology.orchestra.run.smithi150.stdout: osd default data pool replay window = 5 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: mon allow pool delete = true 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: mon cluster log file level = debug 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: debug asserts on shutdown = true 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: mon health detail to clog = false 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: mon host = 172.21.15.78 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: mon client directed command retry = 5 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on pool no app = False 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: ms inject socket failures = 5000 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: fsid = 539007ae-4341-41e5-b60c-c3d4ca69948f 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout:[osd] 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: osd journal size = 100 2024-09-18T00:56:42.387 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: osd scrub load threshold = 5.0 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: osd scrub max interval = 600 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: osd mclock profile = high_recovery_ops 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: osd recover clone overlap = true 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: osd recovery max chunk = 1048576 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug shutdown = true 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug op order = true 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug verify stray on activate = true 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug trim objects = true 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.388 INFO:teuthology.orchestra.run.smithi150.stdout: osd open classes on start = true 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug pg log writeout = true 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: osd deep scrub update digest min age = 30 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: osd map max advance = 10 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: journal zero on create = true 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: filestore ondisk finisher threads = 3 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: filestore apply finisher threads = 3 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: bdev debug aio = true 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug misdirected ops = true 2024-09-18T00:56:42.389 INFO:teuthology.orchestra.run.smithi150.stdout: bluestore block size = 96636764160 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: bluestore compression algorithm = lz4 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: bluestore compression mode = aggressive 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: bluestore fsck on mount = True 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: debug bluefs = 1/20 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: debug bluestore = 1/20 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: debug ms = 1 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: debug osd = 20 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: debug rocksdb = 4/10 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd backfillfull_ratio = 0.85 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd full ratio = 0.9 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd nearfull ratio = 0.8 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: osd failsafe full ratio = 0.95 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: osd objectstore = bluestore 2024-09-18T00:56:42.390 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout:[mgr] 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: debug ms = 1 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: debug mgr = 20 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: debug mon = 20 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: debug auth = 20 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: mon reweight min pgs per osd = 4 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: mon reweight min bytes per osd = 10 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: mgr/telemetry/nag = false 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout:[mon] 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: debug ms = 1 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: debug mon = 20 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: debug paxos = 20 2024-09-18T00:56:42.391 INFO:teuthology.orchestra.run.smithi150.stdout: debug auth = 20 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: mon data avail warn = 5 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: mon mgr mkfs grace = 240 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: mon reweight min pgs per osd = 4 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd reporter subtree level = osd 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd prime pg temp = true 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: mon reweight min bytes per osd = 10 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: auth mon ticket ttl = 660 # 11m 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: auth service ticket ttl = 240 # 4m 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.392 INFO:teuthology.orchestra.run.smithi150.stdout: # don't complain about insecure global_id in the test suite 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: # 1m isn't quite enough 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: mon_down_mkfs_grace = 2m 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: mon_warn_on_filestore_osds = false 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout:[client] 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: rgw cache enabled = true 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: rgw enable ops log = true 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: rgw enable usage log = true 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-18T00:56:42.393 INFO:teuthology.orchestra.run.smithi150.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout: debug rbd = 20 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout: debug rbd_mirror = 15 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout: log to stderr = False 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout: rbd default features = 125 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout: rbd mirror image policy type = simple 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout:[mon.a] 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.0] 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.1] 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.2] 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:42.394 INFO:teuthology.orchestra.run.smithi150.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:42.395 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.3] 2024-09-18T00:56:42.395 INFO:teuthology.orchestra.run.smithi150.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:56:42.395 INFO:teuthology.orchestra.run.smithi150.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:56:42.396 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2024-09-18T00:56:42.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2024-09-18T00:56:42.489 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-18T00:56:42.490 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:42.490 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2024-09-18T00:56:42.505 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:42.505 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2024-09-18T00:56:42.561 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi078.front.sepia.ceph.com 2024-09-18T00:56:42.561 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:42.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/cluster1.keyring 2024-09-18T00:56:42.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-18T00:56:42.634 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:42.634 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2024-09-18T00:56:42.689 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi150.front.sepia.ceph.com 2024-09-18T00:56:42.689 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:42.690 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/ceph/cluster1.keyring 2024-09-18T00:56:42.690 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-09-18T00:56:42.729 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:42.729 DEBUG:teuthology.orchestra.run.smithi150:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2024-09-18T00:56:42.787 INFO:tasks.ceph:Setting up mon nodes... 2024-09-18T00:56:42.787 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-18T00:56:42.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2024-09-18T00:56:42.839 INFO:teuthology.orchestra.run.smithi078.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2024-09-18T00:56:42.842 INFO:tasks.ceph:Setting up mds nodes... 2024-09-18T00:56:42.842 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-18T00:56:42.842 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2024-09-18T00:56:42.883 INFO:teuthology.orchestra.run.smithi078.stdout:creating /etc/ceph/cluster1.client.0.keyring 2024-09-18T00:56:42.895 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2024-09-18T00:56:42.950 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2024-09-18T00:56:42.964 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2024-09-18T00:56:43.008 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2024-09-18T00:56:43.021 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2024-09-18T00:56:43.062 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2024-09-18T00:56:43.075 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2024-09-18T00:56:43.157 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2024-09-18T00:56:43.169 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2024-09-18T00:56:43.250 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2024-09-18T00:56:43.263 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2024-09-18T00:56:43.343 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2024-09-18T00:56:43.356 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2024-09-18T00:56:43.437 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2024-09-18T00:56:43.450 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2024-09-18T00:56:43.530 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2024-09-18T00:56:43.543 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-18T00:56:43.543 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi078.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-18T00:56:43.543 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2024-09-18T00:56:43.570 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-18T00:56:43.570 INFO:tasks.ceph:role: cluster1.osd.0 2024-09-18T00:56:43.570 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-18T00:56:43.571 DEBUG:teuthology.orchestra.run.smithi078:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-18T00:56:43.644 INFO:teuthology.orchestra.run.smithi078.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-18T00:56:43.644 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-18T00:56:43.644 INFO:teuthology.orchestra.run.smithi078.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-18T00:56:43.644 INFO:teuthology.orchestra.run.smithi078.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-18T00:56:43.644 INFO:teuthology.orchestra.run.smithi078.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-18T00:56:43.644 INFO:teuthology.orchestra.run.smithi078.stdout: = sunit=0 swidth=0 blks 2024-09-18T00:56:43.644 INFO:teuthology.orchestra.run.smithi078.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-18T00:56:43.644 INFO:teuthology.orchestra.run.smithi078.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-18T00:56:43.644 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-18T00:56:43.644 INFO:teuthology.orchestra.run.smithi078.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-18T00:56:44.160 INFO:teuthology.orchestra.run.smithi078.stdout:Discarding blocks...Done. 2024-09-18T00:56:44.163 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi078.front.sepia.ceph.com -o noatime 2024-09-18T00:56:44.163 DEBUG:teuthology.orchestra.run.smithi078:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2024-09-18T00:56:44.277 DEBUG:teuthology.orchestra.run.smithi078:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2024-09-18T00:56:44.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2024-09-18T00:56:44.382 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-18T00:56:44.382 INFO:tasks.ceph:role: cluster1.osd.1 2024-09-18T00:56:44.382 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-18T00:56:44.382 DEBUG:teuthology.orchestra.run.smithi078:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-18T00:56:44.455 INFO:teuthology.orchestra.run.smithi078.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-18T00:56:44.455 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-18T00:56:44.456 INFO:teuthology.orchestra.run.smithi078.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-18T00:56:44.456 INFO:teuthology.orchestra.run.smithi078.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-18T00:56:44.456 INFO:teuthology.orchestra.run.smithi078.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-18T00:56:44.456 INFO:teuthology.orchestra.run.smithi078.stdout: = sunit=0 swidth=0 blks 2024-09-18T00:56:44.456 INFO:teuthology.orchestra.run.smithi078.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-18T00:56:44.456 INFO:teuthology.orchestra.run.smithi078.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-18T00:56:44.456 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-18T00:56:44.456 INFO:teuthology.orchestra.run.smithi078.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-18T00:56:44.921 INFO:teuthology.orchestra.run.smithi078.stdout:Discarding blocks...Done. 2024-09-18T00:56:44.923 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi078.front.sepia.ceph.com -o noatime 2024-09-18T00:56:44.923 DEBUG:teuthology.orchestra.run.smithi078:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2024-09-18T00:56:45.021 DEBUG:teuthology.orchestra.run.smithi078:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2024-09-18T00:56:45.049 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2024-09-18T00:56:45.114 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-18T00:56:45.114 INFO:tasks.ceph:role: cluster1.osd.2 2024-09-18T00:56:45.114 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-18T00:56:45.115 DEBUG:teuthology.orchestra.run.smithi078:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-18T00:56:45.186 INFO:teuthology.orchestra.run.smithi078.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-18T00:56:45.186 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-18T00:56:45.186 INFO:teuthology.orchestra.run.smithi078.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-18T00:56:45.186 INFO:teuthology.orchestra.run.smithi078.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-18T00:56:45.186 INFO:teuthology.orchestra.run.smithi078.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-18T00:56:45.186 INFO:teuthology.orchestra.run.smithi078.stdout: = sunit=0 swidth=0 blks 2024-09-18T00:56:45.186 INFO:teuthology.orchestra.run.smithi078.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-18T00:56:45.186 INFO:teuthology.orchestra.run.smithi078.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-18T00:56:45.187 INFO:teuthology.orchestra.run.smithi078.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-18T00:56:45.187 INFO:teuthology.orchestra.run.smithi078.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-18T00:56:45.678 INFO:teuthology.orchestra.run.smithi078.stdout:Discarding blocks...Done. 2024-09-18T00:56:45.680 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi078.front.sepia.ceph.com -o noatime 2024-09-18T00:56:45.680 DEBUG:teuthology.orchestra.run.smithi078:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2024-09-18T00:56:45.818 DEBUG:teuthology.orchestra.run.smithi078:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2024-09-18T00:56:45.848 DEBUG:teuthology.orchestra.run.smithi078:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-18T00:56:45.951 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:45.950+0000 7f27abc19640 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2024-09-18T00:56:45.952 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:45.951+0000 7f27abc19640 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2024-09-18T00:56:45.952 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:45.951+0000 7f27abc19640 -1 bdev(0x5645bc69d800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2024-09-18T00:56:45.953 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:45.951+0000 7f27abc19640 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2024-09-18T00:56:49.365 DEBUG:teuthology.orchestra.run.smithi078:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2024-09-18T00:56:49.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-18T00:56:49.488 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:49.486+0000 7f8ca4a0e640 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2024-09-18T00:56:49.488 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:49.486+0000 7f8ca4a0e640 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2024-09-18T00:56:49.488 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:49.486+0000 7f8ca4a0e640 -1 bdev(0x5603584a7800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2024-09-18T00:56:49.488 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:49.487+0000 7f8ca4a0e640 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2024-09-18T00:56:52.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2024-09-18T00:56:52.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-09-18T00:56:52.526 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:52.524+0000 7f5670dae640 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2024-09-18T00:56:52.526 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:52.524+0000 7f5670dae640 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2024-09-18T00:56:52.526 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:52.525+0000 7f5670dae640 -1 bdev(0x559b1c4e5800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2024-09-18T00:56:52.526 INFO:teuthology.orchestra.run.smithi078.stderr:2024-09-18T00:56:52.525+0000 7f5670dae640 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2024-09-18T00:56:55.689 DEBUG:teuthology.orchestra.run.smithi078:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2024-09-18T00:56:55.714 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-18T00:56:55.715 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:55.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2024-09-18T00:56:55.780 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:55.780 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2024-09-18T00:56:55.845 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:55.845 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2024-09-18T00:56:55.910 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:55.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2024-09-18T00:56:55.976 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:55.976 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2024-09-18T00:56:56.031 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:56.031 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2024-09-18T00:56:56.051 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:56.051 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2024-09-18T00:56:56.110 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:56.110 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2024-09-18T00:56:56.169 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:56.169 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2024-09-18T00:56:56.228 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:56.228 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2024-09-18T00:56:56.285 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:56.285 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2024-09-18T00:56:56.343 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:56.343 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2024-09-18T00:56:56.399 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:56:56.400 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2024-09-18T00:56:56.456 INFO:tasks.ceph:Adding keys to all mons... 2024-09-18T00:56:56.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo tee -a /etc/ceph/cluster1.keyring 2024-09-18T00:56:56.479 INFO:teuthology.orchestra.run.smithi078.stdout:[mgr.x] 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBKJepmk/gFMhAAoxOb0K3tVXmN0ZFlFYpBZg== 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout:[osd.0] 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBNJepm4VS6OBAAGnZw5lyadYcZeutaORyv2w== 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout:[osd.1] 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBRJepm7VIXHRAA77Ge6dEwTUCnMPbiKJd8Hg== 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout:[osd.2] 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBUJepm731bHxAAeCiiPt/CGsjx8+zLYM7DTw== 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout:[client.0] 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBKJepmf9imNBAAGd70odPXCwWGtdR1BxiUqA== 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror] 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBKJepmlWikOBAA4RxHqKWSrV7yNRNnrCpYSA== 2024-09-18T00:56:56.480 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.0] 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBLJepm+gV2ABAAY7eST+q2bk/LCtoPp70xtQ== 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.1] 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBLJepm2YK7AxAAC6u3Le1LVG1gXYw+1KMdaA== 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.2] 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBLJepmNutZCRAA6JtUbX/wfLPgRXSh7RVjIg== 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.3] 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBLJepmfB3tDhAA7712lnLOrPrs6c5NspeRXw== 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.4] 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBLJepm5bd5FBAAuePRnIEI2MphdX60/h+I2w== 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.5] 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBLJepmZ1cSGhAABelv4q/g4r7SyIK4ZrW0vw== 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.6] 2024-09-18T00:56:56.481 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBLJepmliagHxAAUpsrTMtA+yRAkbQ6VR0+VA== 2024-09-18T00:56:56.482 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-18T00:56:56.574 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-18T00:56:56.626 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-18T00:56:56.676 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-18T00:56:56.726 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:56:56.780 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:56:56.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:56:56.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:56:56.939 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:56:56.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:56:57.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:56:57.098 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:56:57.152 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:56:57.206 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-18T00:56:57.206 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2024-09-18T00:56:57.231 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2024-09-18T00:56:57.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2024-09-18T00:56:57.350 DEBUG:teuthology.orchestra.run.smithi078:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2024-09-18T00:56:57.405 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2024-09-18T00:56:57.406 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2024-09-18T00:56:57.406 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2024-09-18T00:56:57.448 INFO:tasks.ceph.cluster1.mon.a:Started 2024-09-18T00:56:57.448 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2024-09-18T00:56:57.448 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2024-09-18T00:56:57.448 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2024-09-18T00:56:57.451 INFO:tasks.ceph.cluster1.mgr.x:Started 2024-09-18T00:56:57.451 INFO:tasks.ceph:Setting crush tunables to default 2024-09-18T00:56:57.451 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster cluster1 osd crush tunables default 2024-09-18T00:56:57.621 INFO:teuthology.orchestra.run.smithi078.stderr:adjusted tunables profile to default 2024-09-18T00:56:57.632 INFO:tasks.ceph:check_enable_crimson: False 2024-09-18T00:56:57.633 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2024-09-18T00:56:57.633 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:57.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2024-09-18T00:56:57.658 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:57.658 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2024-09-18T00:56:57.724 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:56:57.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2024-09-18T00:56:57.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster cluster1 osd new 812a085a-d9f9-4f18-944b-7ac74fa0c228 0 2024-09-18T00:56:57.825 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:56:57.823+0000 7f016b816040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-18T00:56:57.868 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:56:57.866+0000 7f016b816040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-18T00:56:57.999 INFO:teuthology.orchestra.run.smithi078.stdout:0 2024-09-18T00:56:58.007 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:56:58.006+0000 7f016b816040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-18T00:56:58.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster cluster1 osd new 66a760e0-5971-4cbe-a451-985e9d8dd97f 1 2024-09-18T00:56:58.059 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:56:58.058+0000 7f016b816040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-18T00:56:58.226 INFO:teuthology.orchestra.run.smithi078.stdout:1 2024-09-18T00:56:58.226 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:56:58.225+0000 7f016b816040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-18T00:56:58.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster cluster1 osd new bf096562-0f07-440d-aa1c-5ccb6f01b720 2 2024-09-18T00:56:58.341 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:56:58.340+0000 7f016b816040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-18T00:56:58.455 INFO:teuthology.orchestra.run.smithi078.stdout:2 2024-09-18T00:56:58.466 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2024-09-18T00:56:58.466 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2024-09-18T00:56:58.468 INFO:tasks.ceph.cluster1.osd.0:Started 2024-09-18T00:56:58.468 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2024-09-18T00:56:58.468 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2024-09-18T00:56:58.471 INFO:tasks.ceph.cluster1.osd.1:Started 2024-09-18T00:56:58.471 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2024-09-18T00:56:58.471 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2024-09-18T00:56:58.473 INFO:tasks.ceph.cluster1.osd.2:Started 2024-09-18T00:56:58.473 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-18T00:56:58.646 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:56:58.646 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":5,"fsid":"539007ae-4341-41e5-b60c-c3d4ca69948f","created":"2024-09-18T00:56:57.533867+0000","modified":"2024-09-18T00:56:58.454372+0000","last_up_change":"0.000000","last_in_change":"2024-09-18T00:56:58.454372+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"812a085a-d9f9-4f18-944b-7ac74fa0c228","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"66a760e0-5971-4cbe-a451-985e9d8dd97f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"bf096562-0f07-440d-aa1c-5ccb6f01b720","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:56:58.657 INFO:tasks.ceph.ceph_manager.cluster1:[] 2024-09-18T00:56:58.658 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-18T00:56:58.693 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:56:58.691+0000 7f016b816040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-18T00:56:58.745 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:56:58.744+0000 7f016b816040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-18T00:56:58.895 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:56:58.894+0000 7f016b816040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-18T00:56:58.960 DEBUG:teuthology.orchestra.run.smithi078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-18T00:56:59.137 INFO:teuthology.misc.health.smithi078.stdout: 2024-09-18T00:56:59.137 INFO:teuthology.misc.health.smithi078.stdout:{"epoch":5,"fsid":"539007ae-4341-41e5-b60c-c3d4ca69948f","created":"2024-09-18T00:56:57.533867+0000","modified":"2024-09-18T00:56:58.454372+0000","last_up_change":"0.000000","last_in_change":"2024-09-18T00:56:58.454372+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"812a085a-d9f9-4f18-944b-7ac74fa0c228","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"66a760e0-5971-4cbe-a451-985e9d8dd97f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"bf096562-0f07-440d-aa1c-5ccb6f01b720","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:56:59.147 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-18T00:56:59.742 INFO:tasks.ceph.cluster1.osd.1.smithi078.stderr:2024-09-18T00:56:59.740+0000 7f88cc20f640 -1 Falling back to public interface 2024-09-18T00:56:59.886 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:56:59.884+0000 7f016b816040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-18T00:56:59.937 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:56:59.936+0000 7f016b816040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-18T00:56:59.952 INFO:tasks.ceph.cluster1.osd.0.smithi078.stderr:2024-09-18T00:56:59.950+0000 7f7be2cb0640 -1 Falling back to public interface 2024-09-18T00:56:59.985 INFO:tasks.ceph.cluster1.osd.2.smithi078.stderr:2024-09-18T00:56:59.983+0000 7f434b00b640 -1 Falling back to public interface 2024-09-18T00:57:00.304 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:00.302+0000 7f016b816040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-18T00:57:00.386 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:00.385+0000 7f016b816040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-18T00:57:00.492 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:00.490+0000 7f016b816040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-18T00:57:00.538 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:00.537+0000 7f016b816040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-18T00:57:00.652 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:00.650+0000 7f016b816040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-18T00:57:00.704 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:00.702+0000 7f016b816040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-18T00:57:00.749 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:00.747+0000 7f016b816040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-18T00:57:00.862 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:00.860+0000 7f016b816040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-18T00:57:00.911 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:00.910+0000 7f016b816040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-18T00:57:00.962 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:00.961+0000 7f016b816040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-18T00:57:01.029 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:01.028+0000 7f016b816040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-18T00:57:01.073 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:01.072+0000 7f016b816040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-18T00:57:01.118 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:01.116+0000 7f016b816040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-18T00:57:01.765 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:01.764+0000 7f016b816040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-18T00:57:02.973 INFO:tasks.ceph.cluster1.mgr.x.smithi078.stderr:2024-09-18T00:57:02.972+0000 7f010b49b640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-18T00:57:03.559 INFO:tasks.ceph.cluster1.osd.1.smithi078.stderr:2024-09-18T00:57:03.558+0000 7f88cc20f640 -1 osd.1 0 log_to_monitors true 2024-09-18T00:57:03.790 INFO:tasks.ceph.cluster1.osd.2.smithi078.stderr:2024-09-18T00:57:03.789+0000 7f434b00b640 -1 osd.2 0 log_to_monitors true 2024-09-18T00:57:03.997 INFO:tasks.ceph.cluster1.osd.0.smithi078.stderr:2024-09-18T00:57:03.995+0000 7f7be2cb0640 -1 osd.0 0 log_to_monitors true 2024-09-18T00:57:05.444 INFO:tasks.ceph.cluster1.osd.1.smithi078.stderr:2024-09-18T00:57:05.442+0000 7f88c7c1c640 -1 osd.1 0 waiting for initial osdmap 2024-09-18T00:57:05.449 DEBUG:teuthology.orchestra.run.smithi078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-18T00:57:05.462 INFO:tasks.ceph.cluster1.osd.1.smithi078.stderr:2024-09-18T00:57:05.461+0000 7f88bfa3c640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T00:57:05.714 INFO:teuthology.misc.health.smithi078.stdout: 2024-09-18T00:57:05.715 INFO:teuthology.misc.health.smithi078.stdout:{"epoch":7,"fsid":"539007ae-4341-41e5-b60c-c3d4ca69948f","created":"2024-09-18T00:56:57.533867+0000","modified":"2024-09-18T00:57:04.769744+0000","last_up_change":"0.000000","last_in_change":"2024-09-18T00:56:58.454372+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"812a085a-d9f9-4f18-944b-7ac74fa0c228","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"66a760e0-5971-4cbe-a451-985e9d8dd97f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"bf096562-0f07-440d-aa1c-5ccb6f01b720","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:57:05.725 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-18T00:57:06.642 INFO:tasks.ceph.cluster1.osd.0.smithi078.stderr:2024-09-18T00:57:06.640+0000 7f7bdf453640 -1 osd.0 0 waiting for initial osdmap 2024-09-18T00:57:06.655 INFO:tasks.ceph.cluster1.osd.0.smithi078.stderr:2024-09-18T00:57:06.654+0000 7f7bd6a5f640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T00:57:06.658 INFO:tasks.ceph.cluster1.osd.2.smithi078.stderr:2024-09-18T00:57:06.656+0000 7f43468da640 -1 osd.2 0 waiting for initial osdmap 2024-09-18T00:57:06.677 INFO:tasks.ceph.cluster1.osd.2.smithi078.stderr:2024-09-18T00:57:06.676+0000 7f433e6fa640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T00:57:12.029 DEBUG:teuthology.orchestra.run.smithi078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-09-18T00:57:12.323 INFO:teuthology.misc.health.smithi078.stdout: 2024-09-18T00:57:12.324 INFO:teuthology.misc.health.smithi078.stdout:{"epoch":13,"fsid":"539007ae-4341-41e5-b60c-c3d4ca69948f","created":"2024-09-18T00:56:57.533867+0000","modified":"2024-09-18T00:57:10.790360+0000","last_up_change":"2024-09-18T00:57:06.780761+0000","last_in_change":"2024-09-18T00:56:58.454372+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-18T00:57:07.803521+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"812a085a-d9f9-4f18-944b-7ac74fa0c228","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6809","nonce":814730911}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6811","nonce":814730911}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6815","nonce":814730911}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6813","nonce":814730911}]},"public_addr":"172.21.15.78:6809/814730911","cluster_addr":"172.21.15.78:6811/814730911","heartbeat_back_addr":"172.21.15.78:6815/814730911","heartbeat_front_addr":"172.21.15.78:6813/814730911","state":["exists","up"]},{"osd":1,"uuid":"66a760e0-5971-4cbe-a451-985e9d8dd97f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6801","nonce":3340322889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6803","nonce":3340322889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6807","nonce":3340322889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6805","nonce":3340322889}]},"public_addr":"172.21.15.78:6801/3340322889","cluster_addr":"172.21.15.78:6803/3340322889","heartbeat_back_addr":"172.21.15.78:6807/3340322889","heartbeat_front_addr":"172.21.15.78:6805/3340322889","state":["exists","up"]},{"osd":2,"uuid":"bf096562-0f07-440d-aa1c-5ccb6f01b720","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6817","nonce":3900839217}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6819","nonce":3900839217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6823","nonce":3900839217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6821","nonce":3900839217}]},"public_addr":"172.21.15.78:6817/3900839217","cluster_addr":"172.21.15.78:6819/3900839217","heartbeat_back_addr":"172.21.15.78:6823/3900839217","heartbeat_front_addr":"172.21.15.78:6821/3900839217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:05.033037+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:04.521237+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:04.839863+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:57:12.334 DEBUG:teuthology.misc:3 of 3 OSDs are up 2024-09-18T00:57:12.335 INFO:tasks.ceph:Creating RBD pool 2024-09-18T00:57:12.335 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster cluster1 osd pool create rbd 8 2024-09-18T00:57:12.797 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'rbd' created 2024-09-18T00:57:12.812 DEBUG:teuthology.orchestra.run.smithi078:> rbd --cluster cluster1 pool init rbd 2024-09-18T00:57:15.832 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2024-09-18T00:57:15.833 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2024-09-18T00:57:15.833 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2024-09-18T00:57:15.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2024-09-18T00:57:15.834 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-18T00:57:16.157 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:16.168 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":3891294925},{"type":"v1","addr":"172.21.15.78:6825","nonce":3891294925}]},"active_addr":"172.21.15.78:6825/3891294925","active_change":"2024-09-18T00:57:01.766491+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":629895132}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2227542660}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":599752250}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":1743696612}]}]} 2024-09-18T00:57:16.170 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2024-09-18T00:57:16.171 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2024-09-18T00:57:16.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-18T00:57:16.443 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:16.444 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":17,"fsid":"539007ae-4341-41e5-b60c-c3d4ca69948f","created":"2024-09-18T00:56:57.533867+0000","modified":"2024-09-18T00:57:15.818123+0000","last_up_change":"2024-09-18T00:57:06.780761+0000","last_in_change":"2024-09-18T00:56:58.454372+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-18T00:57:07.803521+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-18T00:57:12.626407+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"812a085a-d9f9-4f18-944b-7ac74fa0c228","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6809","nonce":814730911}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6811","nonce":814730911}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6815","nonce":814730911}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6813","nonce":814730911}]},"public_addr":"172.21.15.78:6809/814730911","cluster_addr":"172.21.15.78:6811/814730911","heartbeat_back_addr":"172.21.15.78:6815/814730911","heartbeat_front_addr":"172.21.15.78:6813/814730911","state":["exists","up"]},{"osd":1,"uuid":"66a760e0-5971-4cbe-a451-985e9d8dd97f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6801","nonce":3340322889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6803","nonce":3340322889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6807","nonce":3340322889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6805","nonce":3340322889}]},"public_addr":"172.21.15.78:6801/3340322889","cluster_addr":"172.21.15.78:6803/3340322889","heartbeat_back_addr":"172.21.15.78:6807/3340322889","heartbeat_front_addr":"172.21.15.78:6805/3340322889","state":["exists","up"]},{"osd":2,"uuid":"bf096562-0f07-440d-aa1c-5ccb6f01b720","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6817","nonce":3900839217}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6819","nonce":3900839217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6823","nonce":3900839217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6821","nonce":3900839217}]},"public_addr":"172.21.15.78:6817/3900839217","cluster_addr":"172.21.15.78:6819/3900839217","heartbeat_back_addr":"172.21.15.78:6823/3900839217","heartbeat_front_addr":"172.21.15.78:6821/3900839217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:05.033037+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:04.521237+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:04.839863+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:57:16.456 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2024-09-18T00:57:16.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-18T00:57:16.729 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:16.729 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":17,"fsid":"539007ae-4341-41e5-b60c-c3d4ca69948f","created":"2024-09-18T00:56:57.533867+0000","modified":"2024-09-18T00:57:15.818123+0000","last_up_change":"2024-09-18T00:57:06.780761+0000","last_in_change":"2024-09-18T00:56:58.454372+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-18T00:57:07.803521+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-18T00:57:12.626407+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"812a085a-d9f9-4f18-944b-7ac74fa0c228","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6809","nonce":814730911}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6811","nonce":814730911}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6815","nonce":814730911}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6813","nonce":814730911}]},"public_addr":"172.21.15.78:6809/814730911","cluster_addr":"172.21.15.78:6811/814730911","heartbeat_back_addr":"172.21.15.78:6815/814730911","heartbeat_front_addr":"172.21.15.78:6813/814730911","state":["exists","up"]},{"osd":1,"uuid":"66a760e0-5971-4cbe-a451-985e9d8dd97f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6801","nonce":3340322889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6803","nonce":3340322889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6807","nonce":3340322889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6805","nonce":3340322889}]},"public_addr":"172.21.15.78:6801/3340322889","cluster_addr":"172.21.15.78:6803/3340322889","heartbeat_back_addr":"172.21.15.78:6807/3340322889","heartbeat_front_addr":"172.21.15.78:6805/3340322889","state":["exists","up"]},{"osd":2,"uuid":"bf096562-0f07-440d-aa1c-5ccb6f01b720","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6817","nonce":3900839217}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6819","nonce":3900839217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6823","nonce":3900839217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6821","nonce":3900839217}]},"public_addr":"172.21.15.78:6817/3900839217","cluster_addr":"172.21.15.78:6819/3900839217","heartbeat_back_addr":"172.21.15.78:6823/3900839217","heartbeat_front_addr":"172.21.15.78:6821/3900839217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:05.033037+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:04.521237+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:04.839863+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:57:16.742 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2024-09-18T00:57:16.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2024-09-18T00:57:16.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2024-09-18T00:57:16.882 INFO:teuthology.orchestra.run.smithi078.stdout:38654705669 2024-09-18T00:57:16.883 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2024-09-18T00:57:16.884 INFO:teuthology.orchestra.run.smithi078.stdout:38654705669 2024-09-18T00:57:16.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2024-09-18T00:57:16.885 INFO:teuthology.orchestra.run.smithi078.stdout:34359738374 2024-09-18T00:57:16.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2024-09-18T00:57:17.212 INFO:teuthology.orchestra.run.smithi078.stdout:38654705667 2024-09-18T00:57:17.218 INFO:teuthology.orchestra.run.smithi078.stdout:38654705667 2024-09-18T00:57:17.220 INFO:teuthology.orchestra.run.smithi078.stdout:34359738372 2024-09-18T00:57:17.226 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705669 got 38654705667 for osd.0 2024-09-18T00:57:17.233 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705669 got 38654705667 for osd.2 2024-09-18T00:57:17.234 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738374 got 34359738372 for osd.1 2024-09-18T00:57:18.228 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2024-09-18T00:57:18.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2024-09-18T00:57:18.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2024-09-18T00:57:18.536 INFO:teuthology.orchestra.run.smithi078.stdout:38654705669 2024-09-18T00:57:18.549 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705669 got 38654705669 for osd.0 2024-09-18T00:57:18.549 DEBUG:teuthology.parallel:result is None 2024-09-18T00:57:18.567 INFO:teuthology.orchestra.run.smithi078.stdout:34359738374 2024-09-18T00:57:18.575 INFO:teuthology.orchestra.run.smithi078.stdout:38654705669 2024-09-18T00:57:18.580 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738374 got 34359738374 for osd.1 2024-09-18T00:57:18.580 DEBUG:teuthology.parallel:result is None 2024-09-18T00:57:18.588 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705669 got 38654705669 for osd.2 2024-09-18T00:57:18.588 DEBUG:teuthology.parallel:result is None 2024-09-18T00:57:18.588 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2024-09-18T00:57:18.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T00:57:18.902 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:18.903 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-18T00:57:18.914 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2024-09-18T00:57:17.772363+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81648,"kb_used_data":552,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283033872,"statfs":{"total":289910292480,"available":289826684928,"internally_reserved":0,"allocated":565248,"data_stored":1310194,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.974191"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.827787+0000","last_change":"2024-09-18T00:57:15.827787+0000","last_active":"2024-09-18T00:57:15.827787+0000","last_peered":"2024-09-18T00:57:15.827787+0000","last_clean":"2024-09-18T00:57:15.827787+0000","last_became_active":"2024-09-18T00:57:13.815654+0000","last_became_peered":"2024-09-18T00:57:13.815654+0000","last_unstale":"2024-09-18T00:57:15.827787+0000","last_undegraded":"2024-09-18T00:57:15.827787+0000","last_fullsized":"2024-09-18T00:57:15.827787+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:31.448857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088783099999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.827523+0000","last_change":"2024-09-18T00:57:15.827523+0000","last_active":"2024-09-18T00:57:15.827523+0000","last_peered":"2024-09-18T00:57:15.827523+0000","last_clean":"2024-09-18T00:57:15.827523+0000","last_became_active":"2024-09-18T00:57:13.817123+0000","last_became_peered":"2024-09-18T00:57:13.817123+0000","last_unstale":"2024-09-18T00:57:15.827523+0000","last_undegraded":"2024-09-18T00:57:15.827523+0000","last_fullsized":"2024-09-18T00:57:15.827523+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:30.781530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000818183,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.827552+0000","last_change":"2024-09-18T00:57:15.827552+0000","last_active":"2024-09-18T00:57:15.827552+0000","last_peered":"2024-09-18T00:57:15.827552+0000","last_clean":"2024-09-18T00:57:15.827552+0000","last_became_active":"2024-09-18T00:57:13.816435+0000","last_became_peered":"2024-09-18T00:57:13.816435+0000","last_unstale":"2024-09-18T00:57:15.827552+0000","last_undegraded":"2024-09-18T00:57:15.827552+0000","last_fullsized":"2024-09-18T00:57:15.827552+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T11:59:58.580873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086916700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.827187+0000","last_change":"2024-09-18T00:57:15.827187+0000","last_active":"2024-09-18T00:57:15.827187+0000","last_peered":"2024-09-18T00:57:15.827187+0000","last_clean":"2024-09-18T00:57:15.827187+0000","last_became_active":"2024-09-18T00:57:13.815768+0000","last_became_peered":"2024-09-18T00:57:13.815768+0000","last_unstale":"2024-09-18T00:57:15.827187+0000","last_undegraded":"2024-09-18T00:57:15.827187+0000","last_fullsized":"2024-09-18T00:57:15.827187+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:21:50.803048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000610718,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.826099+0000","last_change":"2024-09-18T00:57:15.825320+0000","last_active":"2024-09-18T00:57:15.826099+0000","last_peered":"2024-09-18T00:57:15.826099+0000","last_clean":"2024-09-18T00:57:15.826099+0000","last_became_active":"2024-09-18T00:57:13.812378+0000","last_became_peered":"2024-09-18T00:57:13.812378+0000","last_unstale":"2024-09-18T00:57:15.826099+0000","last_undegraded":"2024-09-18T00:57:15.826099+0000","last_fullsized":"2024-09-18T00:57:15.826099+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:51:57.842731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001073121,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.822211+0000","last_change":"2024-09-18T00:57:15.822211+0000","last_active":"2024-09-18T00:57:15.822211+0000","last_peered":"2024-09-18T00:57:15.822211+0000","last_clean":"2024-09-18T00:57:15.822211+0000","last_became_active":"2024-09-18T00:57:13.815977+0000","last_became_peered":"2024-09-18T00:57:13.815977+0000","last_unstale":"2024-09-18T00:57:15.822211+0000","last_undegraded":"2024-09-18T00:57:15.822211+0000","last_fullsized":"2024-09-18T00:57:15.822211+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:30:33.665761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040989599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.822309+0000","last_change":"2024-09-18T00:57:15.822309+0000","last_active":"2024-09-18T00:57:15.822309+0000","last_peered":"2024-09-18T00:57:15.822309+0000","last_clean":"2024-09-18T00:57:15.822309+0000","last_became_active":"2024-09-18T00:57:13.816220+0000","last_became_peered":"2024-09-18T00:57:13.816220+0000","last_unstale":"2024-09-18T00:57:15.822309+0000","last_undegraded":"2024-09-18T00:57:15.822309+0000","last_fullsized":"2024-09-18T00:57:15.822309+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:28:51.286449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049982200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.827239+0000","last_change":"2024-09-18T00:57:15.827239+0000","last_active":"2024-09-18T00:57:15.827239+0000","last_peered":"2024-09-18T00:57:15.827239+0000","last_clean":"2024-09-18T00:57:15.827239+0000","last_became_active":"2024-09-18T00:57:13.815577+0000","last_became_peered":"2024-09-18T00:57:13.815577+0000","last_unstale":"2024-09-18T00:57:15.827239+0000","last_undegraded":"2024-09-18T00:57:15.827239+0000","last_fullsized":"2024-09-18T00:57:15.827239+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:42:24.720766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066113999999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'97","reported_seq":147,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.826737+0000","last_change":"2024-09-18T00:57:09.799222+0000","last_active":"2024-09-18T00:57:15.826737+0000","last_peered":"2024-09-18T00:57:15.826737+0000","last_clean":"2024-09-18T00:57:15.826737+0000","last_became_active":"2024-09-18T00:57:09.796934+0000","last_became_peered":"2024-09-18T00:57:09.796934+0000","last_unstale":"2024-09-18T00:57:15.826737+0000","last_undegraded":"2024-09-18T00:57:15.826737+0000","last_fullsized":"2024-09-18T00:57:15.826737+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_clean_scrub_stamp":"2024-09-18T00:57:08.784496+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:37:26.386956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705669,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27172,"kb_used_data":144,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344668,"statfs":{"total":96636764160,"available":96608940032,"internally_reserved":0,"allocated":147456,"data_stored":43140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27236,"kb_used_data":204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344604,"statfs":{"total":96636764160,"available":96608874496,"internally_reserved":0,"allocated":208896,"data_stored":633527,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738374,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27240,"kb_used_data":204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344600,"statfs":{"total":96636764160,"available":96608870400,"internally_reserved":0,"allocated":208896,"data_stored":633527,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T00:57:18.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T00:57:19.186 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:19.187 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-18T00:57:19.198 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2024-09-18T00:57:17.772363+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81648,"kb_used_data":552,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283033872,"statfs":{"total":289910292480,"available":289826684928,"internally_reserved":0,"allocated":565248,"data_stored":1310194,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.974191"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.827787+0000","last_change":"2024-09-18T00:57:15.827787+0000","last_active":"2024-09-18T00:57:15.827787+0000","last_peered":"2024-09-18T00:57:15.827787+0000","last_clean":"2024-09-18T00:57:15.827787+0000","last_became_active":"2024-09-18T00:57:13.815654+0000","last_became_peered":"2024-09-18T00:57:13.815654+0000","last_unstale":"2024-09-18T00:57:15.827787+0000","last_undegraded":"2024-09-18T00:57:15.827787+0000","last_fullsized":"2024-09-18T00:57:15.827787+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:31.448857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088783099999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.827523+0000","last_change":"2024-09-18T00:57:15.827523+0000","last_active":"2024-09-18T00:57:15.827523+0000","last_peered":"2024-09-18T00:57:15.827523+0000","last_clean":"2024-09-18T00:57:15.827523+0000","last_became_active":"2024-09-18T00:57:13.817123+0000","last_became_peered":"2024-09-18T00:57:13.817123+0000","last_unstale":"2024-09-18T00:57:15.827523+0000","last_undegraded":"2024-09-18T00:57:15.827523+0000","last_fullsized":"2024-09-18T00:57:15.827523+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:30.781530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000818183,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.827552+0000","last_change":"2024-09-18T00:57:15.827552+0000","last_active":"2024-09-18T00:57:15.827552+0000","last_peered":"2024-09-18T00:57:15.827552+0000","last_clean":"2024-09-18T00:57:15.827552+0000","last_became_active":"2024-09-18T00:57:13.816435+0000","last_became_peered":"2024-09-18T00:57:13.816435+0000","last_unstale":"2024-09-18T00:57:15.827552+0000","last_undegraded":"2024-09-18T00:57:15.827552+0000","last_fullsized":"2024-09-18T00:57:15.827552+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T11:59:58.580873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086916700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.827187+0000","last_change":"2024-09-18T00:57:15.827187+0000","last_active":"2024-09-18T00:57:15.827187+0000","last_peered":"2024-09-18T00:57:15.827187+0000","last_clean":"2024-09-18T00:57:15.827187+0000","last_became_active":"2024-09-18T00:57:13.815768+0000","last_became_peered":"2024-09-18T00:57:13.815768+0000","last_unstale":"2024-09-18T00:57:15.827187+0000","last_undegraded":"2024-09-18T00:57:15.827187+0000","last_fullsized":"2024-09-18T00:57:15.827187+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:21:50.803048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000610718,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.826099+0000","last_change":"2024-09-18T00:57:15.825320+0000","last_active":"2024-09-18T00:57:15.826099+0000","last_peered":"2024-09-18T00:57:15.826099+0000","last_clean":"2024-09-18T00:57:15.826099+0000","last_became_active":"2024-09-18T00:57:13.812378+0000","last_became_peered":"2024-09-18T00:57:13.812378+0000","last_unstale":"2024-09-18T00:57:15.826099+0000","last_undegraded":"2024-09-18T00:57:15.826099+0000","last_fullsized":"2024-09-18T00:57:15.826099+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:51:57.842731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001073121,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.822211+0000","last_change":"2024-09-18T00:57:15.822211+0000","last_active":"2024-09-18T00:57:15.822211+0000","last_peered":"2024-09-18T00:57:15.822211+0000","last_clean":"2024-09-18T00:57:15.822211+0000","last_became_active":"2024-09-18T00:57:13.815977+0000","last_became_peered":"2024-09-18T00:57:13.815977+0000","last_unstale":"2024-09-18T00:57:15.822211+0000","last_undegraded":"2024-09-18T00:57:15.822211+0000","last_fullsized":"2024-09-18T00:57:15.822211+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:30:33.665761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040989599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.822309+0000","last_change":"2024-09-18T00:57:15.822309+0000","last_active":"2024-09-18T00:57:15.822309+0000","last_peered":"2024-09-18T00:57:15.822309+0000","last_clean":"2024-09-18T00:57:15.822309+0000","last_became_active":"2024-09-18T00:57:13.816220+0000","last_became_peered":"2024-09-18T00:57:13.816220+0000","last_unstale":"2024-09-18T00:57:15.822309+0000","last_undegraded":"2024-09-18T00:57:15.822309+0000","last_fullsized":"2024-09-18T00:57:15.822309+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:28:51.286449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049982200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.827239+0000","last_change":"2024-09-18T00:57:15.827239+0000","last_active":"2024-09-18T00:57:15.827239+0000","last_peered":"2024-09-18T00:57:15.827239+0000","last_clean":"2024-09-18T00:57:15.827239+0000","last_became_active":"2024-09-18T00:57:13.815577+0000","last_became_peered":"2024-09-18T00:57:13.815577+0000","last_unstale":"2024-09-18T00:57:15.827239+0000","last_undegraded":"2024-09-18T00:57:15.827239+0000","last_fullsized":"2024-09-18T00:57:15.827239+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:42:24.720766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066113999999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'97","reported_seq":147,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-18T00:57:15.826737+0000","last_change":"2024-09-18T00:57:09.799222+0000","last_active":"2024-09-18T00:57:15.826737+0000","last_peered":"2024-09-18T00:57:15.826737+0000","last_clean":"2024-09-18T00:57:15.826737+0000","last_became_active":"2024-09-18T00:57:09.796934+0000","last_became_peered":"2024-09-18T00:57:09.796934+0000","last_unstale":"2024-09-18T00:57:15.826737+0000","last_undegraded":"2024-09-18T00:57:15.826737+0000","last_fullsized":"2024-09-18T00:57:15.826737+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_clean_scrub_stamp":"2024-09-18T00:57:08.784496+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:37:26.386956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705669,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27172,"kb_used_data":144,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344668,"statfs":{"total":96636764160,"available":96608940032,"internally_reserved":0,"allocated":147456,"data_stored":43140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27236,"kb_used_data":204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344604,"statfs":{"total":96636764160,"available":96608874496,"internally_reserved":0,"allocated":208896,"data_stored":633527,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738374,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27240,"kb_used_data":204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344600,"statfs":{"total":96636764160,"available":96608870400,"internally_reserved":0,"allocated":208896,"data_stored":633527,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T00:57:19.199 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2024-09-18T00:57:19.199 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2024-09-18T00:57:19.199 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2024-09-18T00:57:19.199 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2024-09-18T00:57:19.508 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:19.508 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-18T00:57:19.520 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2024-09-18T00:57:19.520 INFO:teuthology.run_tasks:Running task ceph... 2024-09-18T00:57:19.530 INFO:tasks.ceph:Creating ceph cluster cluster2... 2024-09-18T00:57:19.530 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-18T00:57:19.530 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686', 'branch': 'squid', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-lz4 policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}', 'email': 'ceph-qa@ceph.com', 'first_in_suite': False, 'job_id': '7909686', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'squid', 'sha1': 'b0d356b98e08f6bc59ef15652f0d0e4298fc83c7'}}, 'owner': 'scheduled_teuthology@teuthology', 'priority': 100, 'repo': 'https://git.ceph.com/ceph.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 1983, 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f', 'sleep_before_teardown': 0, 'subset': '79/128', 'suite': 'rbd', 'suite_branch': 'squid', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph_b0d356b98e08f6bc59ef15652f0d0e4298fc83c7/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph.git', 'suite_sha1': 'b0d356b98e08f6bc59ef15652f0d0e4298fc83c7', 'targets': {'smithi078.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHoBQr+C0uE2Le/lt0B2M0DhCOE+f9NQs6lJvI49ArMnn1SGLqMsYEr4xHrV0h9ypsUL6i4u9Z2dZQH3T3Eifkc=', 'smithi150.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAkxbPf3Tcg64mNNL1gjy0wPGGOSlVPApAHiVtGuSX3ZCnVTGXln5Ad+K5u/FeK/vmT65DuVNC0Z+brnbLwhT6k='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-17_21:16:02', 'tube': 'smithi', 'user': 'teuthology', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-09-18T00:57:19.531 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2024-09-18T00:57:19.533 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2024-09-18T00:57:19.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m0777 -- /var/run/ceph 2024-09-18T00:57:19.589 DEBUG:teuthology.orchestra.run.smithi150:> sudo install -d -m0777 -- /var/run/ceph 2024-09-18T00:57:19.622 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:19.622 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/scratch_devs of=/dev/stdout 2024-09-18T00:57:19.680 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-18T00:57:19.680 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_1 2024-09-18T00:57:19.737 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-18T00:57:19.738 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T00:57:19.738 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-18T00:57:19.738 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T00:57:19.738 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T00:57:19.738 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-09-18 00:56:34.908636590 +0000 2024-09-18T00:57:19.738 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-09-18 00:54:05.659718458 +0000 2024-09-18T00:57:19.738 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-09-18 00:54:05.659718458 +0000 2024-09-18T00:57:19.738 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-09-18 00:54:05.659718458 +0000 2024-09-18T00:57:19.738 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-18T00:57:19.803 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-09-18T00:57:19.803 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-09-18T00:57:19.803 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.00026392 s, 1.9 MB/s 2024-09-18T00:57:19.804 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-18T00:57:19.861 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_2 2024-09-18T00:57:19.919 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-18T00:57:19.919 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T00:57:19.920 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-18T00:57:19.920 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T00:57:19.920 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T00:57:19.920 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-09-18 00:56:34.909636571 +0000 2024-09-18T00:57:19.920 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-09-18 00:54:06.009711819 +0000 2024-09-18T00:57:19.920 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-09-18 00:54:06.009711819 +0000 2024-09-18T00:57:19.920 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-09-18 00:54:06.009711819 +0000 2024-09-18T00:57:19.920 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-18T00:57:19.985 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-09-18T00:57:19.985 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-09-18T00:57:19.985 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000193689 s, 2.6 MB/s 2024-09-18T00:57:19.986 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-18T00:57:20.042 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_3 2024-09-18T00:57:20.100 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-18T00:57:20.100 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T00:57:20.100 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-18T00:57:20.100 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T00:57:20.100 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T00:57:20.101 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-09-18 00:56:34.909636571 +0000 2024-09-18T00:57:20.101 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-09-18 00:54:06.311706090 +0000 2024-09-18T00:57:20.101 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-09-18 00:54:06.311706090 +0000 2024-09-18T00:57:20.101 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-09-18 00:54:06.311706090 +0000 2024-09-18T00:57:20.101 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-18T00:57:20.164 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-09-18T00:57:20.164 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-09-18T00:57:20.164 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.0002377 s, 2.2 MB/s 2024-09-18T00:57:20.166 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-18T00:57:20.221 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_4 2024-09-18T00:57:20.278 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-18T00:57:20.278 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-18T00:57:20.278 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-18T00:57:20.278 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-18T00:57:20.278 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-09-18T00:57:20.279 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-09-18 00:56:34.909636571 +0000 2024-09-18T00:57:20.279 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-09-18 00:54:06.637699906 +0000 2024-09-18T00:57:20.279 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-09-18 00:54:06.637699906 +0000 2024-09-18T00:57:20.279 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-09-18 00:54:06.637699906 +0000 2024-09-18T00:57:20.279 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-18T00:57:20.343 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-09-18T00:57:20.343 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-09-18T00:57:20.343 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000230269 s, 2.2 MB/s 2024-09-18T00:57:20.344 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-18T00:57:20.400 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-18T00:57:20.401 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi150.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-18T00:57:20.401 INFO:tasks.ceph:Generating config... 2024-09-18T00:57:20.402 INFO:tasks.ceph:[client] debug rbd = 20 2024-09-18T00:57:20.402 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2024-09-18T00:57:20.402 INFO:tasks.ceph:[client] log to stderr = False 2024-09-18T00:57:20.402 INFO:tasks.ceph:[client] rbd default features = 125 2024-09-18T00:57:20.402 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2024-09-18T00:57:20.402 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.402 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.402 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.402 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.402 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.403 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.403 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.403 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.403 INFO:tasks.ceph:[global] mon client directed command retry = 5 2024-09-18T00:57:20.403 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-18T00:57:20.403 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2024-09-18T00:57:20.403 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-18T00:57:20.403 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-18T00:57:20.403 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-18T00:57:20.403 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-18T00:57:20.403 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-18T00:57:20.403 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-18T00:57:20.403 INFO:tasks.ceph:[osd] bluestore compression algorithm = lz4 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-18T00:57:20.404 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-18T00:57:20.404 INFO:tasks.ceph:Setting up cluster2.mon.a... 2024-09-18T00:57:20.405 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2024-09-18T00:57:20.481 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster2.keyring 2024-09-18T00:57:20.484 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2024-09-18T00:57:20.567 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-18T00:57:20.632 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '172.21.15.150')] 2024-09-18T00:57:20.632 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.150', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'lz4', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2024-09-18T00:57:20.633 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:20.633 DEBUG:teuthology.orchestra.run.smithi150:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-18T00:57:20.687 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.150 --print /home/ubuntu/cephtest/cluster2.monmap 2024-09-18T00:57:20.767 INFO:teuthology.orchestra.run.smithi150.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2024-09-18T00:57:20.767 INFO:teuthology.orchestra.run.smithi150.stdout:monmaptool: generated fsid 134b3e88-b82a-4f51-8a00-69fd22a05e09 2024-09-18T00:57:20.767 INFO:teuthology.orchestra.run.smithi150.stdout:setting min_mon_release = quincy 2024-09-18T00:57:20.767 INFO:teuthology.orchestra.run.smithi150.stdout:epoch 0 2024-09-18T00:57:20.767 INFO:teuthology.orchestra.run.smithi150.stdout:fsid 134b3e88-b82a-4f51-8a00-69fd22a05e09 2024-09-18T00:57:20.767 INFO:teuthology.orchestra.run.smithi150.stdout:last_changed 2024-09-18T00:57:20.767213+0000 2024-09-18T00:57:20.768 INFO:teuthology.orchestra.run.smithi150.stdout:created 2024-09-18T00:57:20.767213+0000 2024-09-18T00:57:20.768 INFO:teuthology.orchestra.run.smithi150.stdout:min_mon_release 17 (quincy) 2024-09-18T00:57:20.768 INFO:teuthology.orchestra.run.smithi150.stdout:election_strategy: 1 2024-09-18T00:57:20.768 INFO:teuthology.orchestra.run.smithi150.stdout:0: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.a 2024-09-18T00:57:20.768 INFO:teuthology.orchestra.run.smithi150.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2024-09-18T00:57:20.769 DEBUG:teuthology.orchestra.run.smithi150:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-18T00:57:20.823 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 134b3e88-b82a-4f51-8a00-69fd22a05e09... 2024-09-18T00:57:20.824 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2024-09-18T00:57:20.826 DEBUG:teuthology.orchestra.run.smithi150:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2024-09-18T00:57:20.877 INFO:teuthology.orchestra.run.smithi078.stdout:[global] 2024-09-18T00:57:20.877 INFO:teuthology.orchestra.run.smithi078.stdout: chdir = "" 2024-09-18T00:57:20.877 INFO:teuthology.orchestra.run.smithi078.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-18T00:57:20.877 INFO:teuthology.orchestra.run.smithi078.stdout: auth supported = cephx 2024-09-18T00:57:20.877 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.877 INFO:teuthology.orchestra.run.smithi078.stdout: filestore xattr use omap = true 2024-09-18T00:57:20.877 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.877 INFO:teuthology.orchestra.run.smithi078.stdout: mon clock drift allowed = 1.000 2024-09-18T00:57:20.877 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.877 INFO:teuthology.orchestra.run.smithi078.stdout: osd crush chooseleaf type = 0 2024-09-18T00:57:20.877 INFO:teuthology.orchestra.run.smithi078.stdout: auth debug = true 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: ms die on old message = true 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: ms die on bug = true 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: mon pg warn max object skew = 0 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: # disable pg_autoscaler by default for new pools 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: osd pool default size = 2 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd allow primary affinity = true 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd allow pg remap = true 2024-09-18T00:57:20.878 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on legacy crush tunables = false 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on crush straw calc version zero = false 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on no sortbitwise = false 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on osd down out interval zero = false 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on too few osds = false 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: mon_allow_pool_size_one = true 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: osd default data pool replay window = 5 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: mon allow pool delete = true 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.879 INFO:teuthology.orchestra.run.smithi078.stdout: mon cluster log file level = debug 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: debug asserts on shutdown = true 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: mon health detail to clog = false 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: mon host = 172.21.15.150 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: mon client directed command retry = 5 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: mon warn on pool no app = False 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: ms inject socket failures = 5000 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: fsid = 134b3e88-b82a-4f51-8a00-69fd22a05e09 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout:[osd] 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: osd journal size = 100 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: osd scrub load threshold = 5.0 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: osd scrub max interval = 600 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: osd mclock profile = high_recovery_ops 2024-09-18T00:57:20.880 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: osd recover clone overlap = true 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: osd recovery max chunk = 1048576 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug shutdown = true 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug op order = true 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug verify stray on activate = true 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug trim objects = true 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: osd open classes on start = true 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug pg log writeout = true 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: osd deep scrub update digest min age = 30 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: osd map max advance = 10 2024-09-18T00:57:20.881 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: journal zero on create = true 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: filestore ondisk finisher threads = 3 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: filestore apply finisher threads = 3 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: bdev debug aio = true 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: osd debug misdirected ops = true 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore block size = 96636764160 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore compression algorithm = lz4 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore compression mode = aggressive 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: bluestore fsck on mount = True 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: debug bluefs = 1/20 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: debug bluestore = 1/20 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: debug ms = 1 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: debug osd = 20 2024-09-18T00:57:20.882 INFO:teuthology.orchestra.run.smithi078.stdout: debug rocksdb = 4/10 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd backfillfull_ratio = 0.85 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd full ratio = 0.9 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd nearfull ratio = 0.8 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: osd failsafe full ratio = 0.95 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: osd objectstore = bluestore 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout:[mgr] 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: debug ms = 1 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: debug mgr = 20 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: debug mon = 20 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: debug auth = 20 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min pgs per osd = 4 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min bytes per osd = 10 2024-09-18T00:57:20.883 INFO:teuthology.orchestra.run.smithi078.stdout: mgr/telemetry/nag = false 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout:[mon] 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: debug ms = 1 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: debug mon = 20 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: debug paxos = 20 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: debug auth = 20 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: mon data avail warn = 5 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: mon mgr mkfs grace = 240 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min pgs per osd = 4 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd reporter subtree level = osd 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: mon osd prime pg temp = true 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: mon reweight min bytes per osd = 10 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.884 INFO:teuthology.orchestra.run.smithi078.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: auth mon ticket ttl = 660 # 11m 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: auth service ticket ttl = 240 # 4m 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: # don't complain about insecure global_id in the test suite 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: # 1m isn't quite enough 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: mon_down_mkfs_grace = 2m 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: mon_warn_on_filestore_osds = false 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout:[client] 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: rgw cache enabled = true 2024-09-18T00:57:20.885 INFO:teuthology.orchestra.run.smithi078.stdout: rgw enable ops log = true 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: rgw enable usage log = true 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: debug rbd = 20 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: debug rbd_mirror = 15 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: log to stderr = False 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: rbd default features = 125 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: rbd mirror image policy type = simple 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout:[mon.a] 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.0] 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.1] 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.886 INFO:teuthology.orchestra.run.smithi078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.887 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.2] 2024-09-18T00:57:20.887 INFO:teuthology.orchestra.run.smithi078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.887 INFO:teuthology.orchestra.run.smithi078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.887 INFO:teuthology.orchestra.run.smithi078.stdout:[client.mirror.3] 2024-09-18T00:57:20.887 INFO:teuthology.orchestra.run.smithi078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.887 INFO:teuthology.orchestra.run.smithi078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.906 INFO:teuthology.orchestra.run.smithi150.stdout:[global] 2024-09-18T00:57:20.907 INFO:teuthology.orchestra.run.smithi150.stdout: chdir = "" 2024-09-18T00:57:20.907 INFO:teuthology.orchestra.run.smithi150.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-18T00:57:20.907 INFO:teuthology.orchestra.run.smithi150.stdout: auth supported = cephx 2024-09-18T00:57:20.907 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.907 INFO:teuthology.orchestra.run.smithi150.stdout: filestore xattr use omap = true 2024-09-18T00:57:20.907 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.907 INFO:teuthology.orchestra.run.smithi150.stdout: mon clock drift allowed = 1.000 2024-09-18T00:57:20.907 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.907 INFO:teuthology.orchestra.run.smithi150.stdout: osd crush chooseleaf type = 0 2024-09-18T00:57:20.907 INFO:teuthology.orchestra.run.smithi150.stdout: auth debug = true 2024-09-18T00:57:20.907 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: ms die on old message = true 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: ms die on bug = true 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: mon pg warn max object skew = 0 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: # disable pg_autoscaler by default for new pools 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: osd pool default size = 2 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd allow primary affinity = true 2024-09-18T00:57:20.908 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd allow pg remap = true 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on legacy crush tunables = false 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on crush straw calc version zero = false 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on no sortbitwise = false 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on osd down out interval zero = false 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on too few osds = false 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: mon_allow_pool_size_one = true 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: osd default data pool replay window = 5 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.909 INFO:teuthology.orchestra.run.smithi150.stdout: mon allow pool delete = true 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: mon cluster log file level = debug 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: debug asserts on shutdown = true 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: mon health detail to clog = false 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: mon host = 172.21.15.150 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: mon client directed command retry = 5 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: mon warn on pool no app = False 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: ms inject socket failures = 5000 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: fsid = 134b3e88-b82a-4f51-8a00-69fd22a05e09 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout:[osd] 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: osd journal size = 100 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: osd scrub load threshold = 5.0 2024-09-18T00:57:20.910 INFO:teuthology.orchestra.run.smithi150.stdout: osd scrub max interval = 600 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: osd mclock profile = high_recovery_ops 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: osd recover clone overlap = true 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: osd recovery max chunk = 1048576 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug shutdown = true 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug op order = true 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug verify stray on activate = true 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug trim objects = true 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: osd open classes on start = true 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug pg log writeout = true 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.911 INFO:teuthology.orchestra.run.smithi150.stdout: osd deep scrub update digest min age = 30 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: osd map max advance = 10 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: journal zero on create = true 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: filestore ondisk finisher threads = 3 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: filestore apply finisher threads = 3 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: bdev debug aio = true 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: osd debug misdirected ops = true 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: bluestore block size = 96636764160 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: bluestore compression algorithm = lz4 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: bluestore compression mode = aggressive 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: bluestore fsck on mount = True 2024-09-18T00:57:20.912 INFO:teuthology.orchestra.run.smithi150.stdout: debug bluefs = 1/20 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: debug bluestore = 1/20 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: debug ms = 1 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: debug osd = 20 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: debug rocksdb = 4/10 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd backfillfull_ratio = 0.85 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd full ratio = 0.9 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd nearfull ratio = 0.8 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: osd failsafe full ratio = 0.95 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: osd objectstore = bluestore 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout:[mgr] 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: debug ms = 1 2024-09-18T00:57:20.913 INFO:teuthology.orchestra.run.smithi150.stdout: debug mgr = 20 2024-09-18T00:57:20.914 INFO:teuthology.orchestra.run.smithi150.stdout: debug mon = 20 2024-09-18T00:57:20.914 INFO:teuthology.orchestra.run.smithi150.stdout: debug auth = 20 2024-09-18T00:57:20.914 INFO:teuthology.orchestra.run.smithi150.stdout: mon reweight min pgs per osd = 4 2024-09-18T00:57:20.914 INFO:teuthology.orchestra.run.smithi150.stdout: mon reweight min bytes per osd = 10 2024-09-18T00:57:20.914 INFO:teuthology.orchestra.run.smithi150.stdout: mgr/telemetry/nag = false 2024-09-18T00:57:20.914 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.914 INFO:teuthology.orchestra.run.smithi150.stdout:[mon] 2024-09-18T00:57:20.914 INFO:teuthology.orchestra.run.smithi150.stdout: debug ms = 1 2024-09-18T00:57:20.914 INFO:teuthology.orchestra.run.smithi150.stdout: debug mon = 20 2024-09-18T00:57:20.914 INFO:teuthology.orchestra.run.smithi150.stdout: debug paxos = 20 2024-09-18T00:57:20.914 INFO:teuthology.orchestra.run.smithi150.stdout: debug auth = 20 2024-09-18T00:57:20.915 INFO:teuthology.orchestra.run.smithi150.stdout: mon data avail warn = 5 2024-09-18T00:57:20.915 INFO:teuthology.orchestra.run.smithi150.stdout: mon mgr mkfs grace = 240 2024-09-18T00:57:20.915 INFO:teuthology.orchestra.run.smithi150.stdout: mon reweight min pgs per osd = 4 2024-09-18T00:57:20.915 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd reporter subtree level = osd 2024-09-18T00:57:20.915 INFO:teuthology.orchestra.run.smithi150.stdout: mon osd prime pg temp = true 2024-09-18T00:57:20.915 INFO:teuthology.orchestra.run.smithi150.stdout: mon reweight min bytes per osd = 10 2024-09-18T00:57:20.915 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.915 INFO:teuthology.orchestra.run.smithi150.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-18T00:57:20.915 INFO:teuthology.orchestra.run.smithi150.stdout: auth mon ticket ttl = 660 # 11m 2024-09-18T00:57:20.915 INFO:teuthology.orchestra.run.smithi150.stdout: auth service ticket ttl = 240 # 4m 2024-09-18T00:57:20.915 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.916 INFO:teuthology.orchestra.run.smithi150.stdout: # don't complain about insecure global_id in the test suite 2024-09-18T00:57:20.916 INFO:teuthology.orchestra.run.smithi150.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-18T00:57:20.916 INFO:teuthology.orchestra.run.smithi150.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-18T00:57:20.916 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.916 INFO:teuthology.orchestra.run.smithi150.stdout: # 1m isn't quite enough 2024-09-18T00:57:20.916 INFO:teuthology.orchestra.run.smithi150.stdout: mon_down_mkfs_grace = 2m 2024-09-18T00:57:20.916 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.916 INFO:teuthology.orchestra.run.smithi150.stdout: mon_warn_on_filestore_osds = false 2024-09-18T00:57:20.916 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:20.916 INFO:teuthology.orchestra.run.smithi150.stdout:[client] 2024-09-18T00:57:20.916 INFO:teuthology.orchestra.run.smithi150.stdout: rgw cache enabled = true 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout: rgw enable ops log = true 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout: rgw enable usage log = true 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout: debug rbd = 20 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout: debug rbd_mirror = 15 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout: log to stderr = False 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout: rbd default features = 125 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout: rbd mirror image policy type = simple 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout:[mon.a] 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.0] 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.917 INFO:teuthology.orchestra.run.smithi150.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.918 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.1] 2024-09-18T00:57:20.918 INFO:teuthology.orchestra.run.smithi150.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.918 INFO:teuthology.orchestra.run.smithi150.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.918 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.2] 2024-09-18T00:57:20.918 INFO:teuthology.orchestra.run.smithi150.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.918 INFO:teuthology.orchestra.run.smithi150.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.918 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.3] 2024-09-18T00:57:20.918 INFO:teuthology.orchestra.run.smithi150.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-09-18T00:57:20.918 INFO:teuthology.orchestra.run.smithi150.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-09-18T00:57:20.919 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2024-09-18T00:57:20.919 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2024-09-18T00:57:21.015 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-18T00:57:21.015 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:21.015 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2024-09-18T00:57:21.031 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:21.031 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2024-09-18T00:57:21.086 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi078.front.sepia.ceph.com 2024-09-18T00:57:21.087 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:57:21.087 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/cluster2.keyring 2024-09-18T00:57:21.087 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-18T00:57:21.128 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:57:21.128 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2024-09-18T00:57:21.187 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi150.front.sepia.ceph.com 2024-09-18T00:57:21.187 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:21.187 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/ceph/cluster2.keyring 2024-09-18T00:57:21.187 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-09-18T00:57:21.222 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:21.222 DEBUG:teuthology.orchestra.run.smithi150:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2024-09-18T00:57:21.278 INFO:tasks.ceph:Setting up mon nodes... 2024-09-18T00:57:21.278 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-18T00:57:21.279 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2024-09-18T00:57:21.338 INFO:teuthology.orchestra.run.smithi078.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2024-09-18T00:57:21.341 INFO:tasks.ceph:Setting up mds nodes... 2024-09-18T00:57:21.341 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-18T00:57:21.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2024-09-18T00:57:21.386 INFO:teuthology.orchestra.run.smithi078.stdout:creating /etc/ceph/cluster2.client.0.keyring 2024-09-18T00:57:21.399 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2024-09-18T00:57:21.439 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2024-09-18T00:57:21.452 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2024-09-18T00:57:21.533 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2024-09-18T00:57:21.546 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2024-09-18T00:57:21.627 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2024-09-18T00:57:21.641 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2024-09-18T00:57:21.721 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2024-09-18T00:57:21.734 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2024-09-18T00:57:21.814 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2024-09-18T00:57:21.827 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2024-09-18T00:57:21.908 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2024-09-18T00:57:21.921 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2024-09-18T00:57:22.002 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2024-09-18T00:57:22.015 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2024-09-18T00:57:22.096 INFO:teuthology.orchestra.run.smithi150.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2024-09-18T00:57:22.108 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-18T00:57:22.109 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi078.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@smithi150.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2024-09-18T00:57:22.109 DEBUG:teuthology.orchestra.run.smithi150:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2024-09-18T00:57:22.174 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-18T00:57:22.174 INFO:tasks.ceph:role: cluster2.osd.0 2024-09-18T00:57:22.174 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi150.front.sepia.ceph.com 2024-09-18T00:57:22.174 DEBUG:teuthology.orchestra.run.smithi150:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-18T00:57:22.246 INFO:teuthology.orchestra.run.smithi150.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-18T00:57:22.246 INFO:teuthology.orchestra.run.smithi150.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-18T00:57:22.246 INFO:teuthology.orchestra.run.smithi150.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-18T00:57:22.246 INFO:teuthology.orchestra.run.smithi150.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-18T00:57:22.246 INFO:teuthology.orchestra.run.smithi150.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-18T00:57:22.246 INFO:teuthology.orchestra.run.smithi150.stdout: = sunit=0 swidth=0 blks 2024-09-18T00:57:22.246 INFO:teuthology.orchestra.run.smithi150.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-18T00:57:22.246 INFO:teuthology.orchestra.run.smithi150.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-18T00:57:22.246 INFO:teuthology.orchestra.run.smithi150.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-18T00:57:22.247 INFO:teuthology.orchestra.run.smithi150.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-18T00:57:22.751 INFO:teuthology.orchestra.run.smithi150.stdout:Discarding blocks...Done. 2024-09-18T00:57:22.753 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi150.front.sepia.ceph.com -o noatime 2024-09-18T00:57:22.753 DEBUG:teuthology.orchestra.run.smithi150:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2024-09-18T00:57:22.860 DEBUG:teuthology.orchestra.run.smithi150:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2024-09-18T00:57:22.892 DEBUG:teuthology.orchestra.run.smithi150:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2024-09-18T00:57:22.958 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-18T00:57:22.958 INFO:tasks.ceph:role: cluster2.osd.1 2024-09-18T00:57:22.958 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi150.front.sepia.ceph.com 2024-09-18T00:57:22.958 DEBUG:teuthology.orchestra.run.smithi150:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-18T00:57:23.031 INFO:teuthology.orchestra.run.smithi150.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-18T00:57:23.031 INFO:teuthology.orchestra.run.smithi150.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-18T00:57:23.031 INFO:teuthology.orchestra.run.smithi150.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-18T00:57:23.031 INFO:teuthology.orchestra.run.smithi150.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-18T00:57:23.032 INFO:teuthology.orchestra.run.smithi150.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-18T00:57:23.032 INFO:teuthology.orchestra.run.smithi150.stdout: = sunit=0 swidth=0 blks 2024-09-18T00:57:23.032 INFO:teuthology.orchestra.run.smithi150.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-18T00:57:23.032 INFO:teuthology.orchestra.run.smithi150.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-18T00:57:23.032 INFO:teuthology.orchestra.run.smithi150.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-18T00:57:23.032 INFO:teuthology.orchestra.run.smithi150.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-18T00:57:23.497 INFO:teuthology.orchestra.run.smithi150.stdout:Discarding blocks...Done. 2024-09-18T00:57:23.500 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi150.front.sepia.ceph.com -o noatime 2024-09-18T00:57:23.500 DEBUG:teuthology.orchestra.run.smithi150:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2024-09-18T00:57:23.585 DEBUG:teuthology.orchestra.run.smithi150:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2024-09-18T00:57:23.613 DEBUG:teuthology.orchestra.run.smithi150:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2024-09-18T00:57:23.679 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-09-18T00:57:23.679 INFO:tasks.ceph:role: cluster2.osd.2 2024-09-18T00:57:23.679 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi150.front.sepia.ceph.com 2024-09-18T00:57:23.679 DEBUG:teuthology.orchestra.run.smithi150:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-18T00:57:23.751 INFO:teuthology.orchestra.run.smithi150.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-18T00:57:23.751 INFO:teuthology.orchestra.run.smithi150.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-18T00:57:23.752 INFO:teuthology.orchestra.run.smithi150.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-18T00:57:23.752 INFO:teuthology.orchestra.run.smithi150.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-18T00:57:23.752 INFO:teuthology.orchestra.run.smithi150.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-18T00:57:23.752 INFO:teuthology.orchestra.run.smithi150.stdout: = sunit=0 swidth=0 blks 2024-09-18T00:57:23.752 INFO:teuthology.orchestra.run.smithi150.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-18T00:57:23.752 INFO:teuthology.orchestra.run.smithi150.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-18T00:57:23.752 INFO:teuthology.orchestra.run.smithi150.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-18T00:57:23.752 INFO:teuthology.orchestra.run.smithi150.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-18T00:57:24.243 INFO:teuthology.orchestra.run.smithi150.stdout:Discarding blocks...Done. 2024-09-18T00:57:24.245 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi150.front.sepia.ceph.com -o noatime 2024-09-18T00:57:24.245 DEBUG:teuthology.orchestra.run.smithi150:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2024-09-18T00:57:24.379 DEBUG:teuthology.orchestra.run.smithi150:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2024-09-18T00:57:24.411 DEBUG:teuthology.orchestra.run.smithi150:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-18T00:57:24.503 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:24.501+0000 7f3aa3a1f640 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2024-09-18T00:57:24.504 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:24.502+0000 7f3aa3a1f640 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2024-09-18T00:57:24.504 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:24.502+0000 7f3aa3a1f640 -1 bdev(0x55fe0dedb800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2024-09-18T00:57:24.504 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:24.502+0000 7f3aa3a1f640 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2024-09-18T00:57:27.913 DEBUG:teuthology.orchestra.run.smithi150:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2024-09-18T00:57:27.939 DEBUG:teuthology.orchestra.run.smithi150:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-18T00:57:28.026 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:28.023+0000 7ff688330640 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2024-09-18T00:57:28.026 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:28.024+0000 7ff688330640 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2024-09-18T00:57:28.026 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:28.024+0000 7ff688330640 -1 bdev(0x55f69b8a9800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2024-09-18T00:57:28.026 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:28.024+0000 7ff688330640 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2024-09-18T00:57:30.932 DEBUG:teuthology.orchestra.run.smithi150:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2024-09-18T00:57:30.958 DEBUG:teuthology.orchestra.run.smithi150:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-09-18T00:57:31.044 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:31.042+0000 7f024ca15640 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2024-09-18T00:57:31.044 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:31.042+0000 7f024ca15640 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2024-09-18T00:57:31.044 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:31.042+0000 7f024ca15640 -1 bdev(0x55bbf92b5800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2024-09-18T00:57:31.045 INFO:teuthology.orchestra.run.smithi150.stderr:2024-09-18T00:57:31.043+0000 7f024ca15640 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2024-09-18T00:57:33.953 DEBUG:teuthology.orchestra.run.smithi150:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2024-09-18T00:57:33.980 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-18T00:57:33.981 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:57:33.981 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2024-09-18T00:57:34.011 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:34.011 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2024-09-18T00:57:34.047 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:34.047 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2024-09-18T00:57:34.114 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:34.114 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2024-09-18T00:57:34.179 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-18T00:57:34.179 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2024-09-18T00:57:34.198 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:34.198 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2024-09-18T00:57:34.235 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:34.235 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2024-09-18T00:57:34.291 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:34.291 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2024-09-18T00:57:34.346 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:34.347 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2024-09-18T00:57:34.402 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:34.402 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2024-09-18T00:57:34.458 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:34.458 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2024-09-18T00:57:34.514 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:34.514 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2024-09-18T00:57:34.570 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:34.570 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2024-09-18T00:57:34.629 INFO:tasks.ceph:Adding keys to all mons... 2024-09-18T00:57:34.629 DEBUG:teuthology.orchestra.run.smithi150:> sudo tee -a /etc/ceph/cluster2.keyring 2024-09-18T00:57:34.692 INFO:teuthology.orchestra.run.smithi150.stdout:[mgr.x] 2024-09-18T00:57:34.692 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmbGwrFBAA+lOs83jGRURTL0TzptBTyA== 2024-09-18T00:57:34.693 INFO:teuthology.orchestra.run.smithi150.stdout:[osd.0] 2024-09-18T00:57:34.693 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQB0JepmxWgBHhAADiK/F6bUuye6Yc23oBsavw== 2024-09-18T00:57:34.693 INFO:teuthology.orchestra.run.smithi150.stdout:[osd.1] 2024-09-18T00:57:34.693 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQB4JepmDxaBARAA4TOXUabAGbYVFLx09+ROWg== 2024-09-18T00:57:34.693 INFO:teuthology.orchestra.run.smithi150.stdout:[osd.2] 2024-09-18T00:57:34.693 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQB7Jepmx+GgAhAAAIxdcEZ9kM75WaIvnPOaXw== 2024-09-18T00:57:34.693 INFO:teuthology.orchestra.run.smithi150.stdout:[client.0] 2024-09-18T00:57:34.693 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmSbACFxAA5aWRZEqMPpWFjZJNG0lsKA== 2024-09-18T00:57:34.693 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror] 2024-09-18T00:57:34.693 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmdGA2GhAAgyv5NSZlHf9LaVXZ4UDltw== 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.0] 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmFz/MHxAAqJu6cWplUA4Mjmd6pkoKLA== 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.1] 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmuY5lJRAAZTS7qN1Weg4c8TXjPtOVLw== 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.2] 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmbDgAKxAA6bfCvEaFmN8fGnAdi1k/Eg== 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.3] 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmgUGPMBAAaRcvmTZPA8XI8vsyRNxtcQ== 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.4] 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmxr4mNhAAaUg3m1R4kVs8OBzh94RRQQ== 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.5] 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQByJepmgGAnABAA488tVG6tKzRiWrHckEspXg== 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.6] 2024-09-18T00:57:34.694 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQByJepmIQO4BRAAflZ0dDQEk2eCd0uM4bPTYA== 2024-09-18T00:57:34.695 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-18T00:57:34.788 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-18T00:57:34.839 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-18T00:57:34.890 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-18T00:57:34.941 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:57:34.995 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:57:35.049 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:57:35.104 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:57:35.159 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:57:35.213 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:57:35.267 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:57:35.321 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:57:35.375 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-09-18T00:57:35.430 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-18T00:57:35.431 DEBUG:teuthology.orchestra.run.smithi150:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2024-09-18T00:57:35.455 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2024-09-18T00:57:35.553 DEBUG:teuthology.orchestra.run.smithi150:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2024-09-18T00:57:35.578 DEBUG:teuthology.orchestra.run.smithi150:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2024-09-18T00:57:35.633 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2024-09-18T00:57:35.634 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2024-09-18T00:57:35.634 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2024-09-18T00:57:35.676 INFO:tasks.ceph.cluster2.mon.a:Started 2024-09-18T00:57:35.677 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2024-09-18T00:57:35.677 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2024-09-18T00:57:35.677 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2024-09-18T00:57:35.680 INFO:tasks.ceph.cluster2.mgr.x:Started 2024-09-18T00:57:35.680 INFO:tasks.ceph:Setting crush tunables to default 2024-09-18T00:57:35.680 DEBUG:teuthology.orchestra.run.smithi150:> sudo ceph --cluster cluster2 osd crush tunables default 2024-09-18T00:57:35.851 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:35.850+0000 7fead791f040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-18T00:57:35.851 INFO:teuthology.orchestra.run.smithi150.stderr:adjusted tunables profile to default 2024-09-18T00:57:35.861 INFO:tasks.ceph:check_enable_crimson: False 2024-09-18T00:57:35.862 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2024-09-18T00:57:35.862 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:35.862 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2024-09-18T00:57:35.887 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:35.887 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2024-09-18T00:57:35.890 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:35.890+0000 7fead791f040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-18T00:57:35.952 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:57:35.952 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2024-09-18T00:57:36.017 DEBUG:teuthology.orchestra.run.smithi150:> sudo ceph --cluster cluster2 osd new dda8f346-bbdb-42fd-876f-c6cdec5d1826 0 2024-09-18T00:57:36.019 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:36.018+0000 7fead791f040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-18T00:57:36.068 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:36.067+0000 7fead791f040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-18T00:57:36.175 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:36.175+0000 7fead791f040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-18T00:57:36.232 INFO:teuthology.orchestra.run.smithi150.stdout:0 2024-09-18T00:57:36.243 DEBUG:teuthology.orchestra.run.smithi150:> sudo ceph --cluster cluster2 osd new 32875857-ffe6-4037-be43-509984a1afdf 1 2024-09-18T00:57:36.268 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:36.268+0000 7fead791f040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-18T00:57:36.417 INFO:teuthology.orchestra.run.smithi150.stdout:1 2024-09-18T00:57:36.428 DEBUG:teuthology.orchestra.run.smithi150:> sudo ceph --cluster cluster2 osd new 0095f5ec-4942-4407-a217-6923831ae2ae 2 2024-09-18T00:57:36.525 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:36.525+0000 7fead791f040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-18T00:57:36.569 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:36.568+0000 7fead791f040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-18T00:57:36.605 INFO:teuthology.orchestra.run.smithi150.stdout:2 2024-09-18T00:57:36.616 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2024-09-18T00:57:36.616 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2024-09-18T00:57:36.618 INFO:tasks.ceph.cluster2.osd.0:Started 2024-09-18T00:57:36.618 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2024-09-18T00:57:36.618 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2024-09-18T00:57:36.620 INFO:tasks.ceph.cluster2.osd.1:Started 2024-09-18T00:57:36.620 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2024-09-18T00:57:36.621 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2024-09-18T00:57:36.622 INFO:tasks.ceph.cluster2.osd.2:Started 2024-09-18T00:57:36.623 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-18T00:57:36.683 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:36.682+0000 7fead791f040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-18T00:57:36.789 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:36.790 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":5,"fsid":"134b3e88-b82a-4f51-8a00-69fd22a05e09","created":"2024-09-18T00:57:35.765780+0000","modified":"2024-09-18T00:57:36.604173+0000","last_up_change":"0.000000","last_in_change":"2024-09-18T00:57:36.604173+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"dda8f346-bbdb-42fd-876f-c6cdec5d1826","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"32875857-ffe6-4037-be43-509984a1afdf","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"0095f5ec-4942-4407-a217-6923831ae2ae","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:57:36.801 INFO:tasks.ceph.ceph_manager.cluster2:[] 2024-09-18T00:57:36.801 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-18T00:57:37.104 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-18T00:57:37.261 INFO:teuthology.misc.health.smithi150.stdout: 2024-09-18T00:57:37.261 INFO:teuthology.misc.health.smithi150.stdout:{"epoch":5,"fsid":"134b3e88-b82a-4f51-8a00-69fd22a05e09","created":"2024-09-18T00:57:35.765780+0000","modified":"2024-09-18T00:57:36.604173+0000","last_up_change":"0.000000","last_in_change":"2024-09-18T00:57:36.604173+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"dda8f346-bbdb-42fd-876f-c6cdec5d1826","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"32875857-ffe6-4037-be43-509984a1afdf","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"0095f5ec-4942-4407-a217-6923831ae2ae","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:57:37.270 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-18T00:57:37.371 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:37.370+0000 7fead791f040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-18T00:57:37.414 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:37.413+0000 7fead791f040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-18T00:57:37.720 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:37.719+0000 7fead791f040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-18T00:57:37.802 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:37.801+0000 7fead791f040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-18T00:57:37.818 INFO:tasks.ceph.cluster2.osd.0.smithi150.stderr:2024-09-18T00:57:37.817+0000 7f8991a0d640 -1 Falling back to public interface 2024-09-18T00:57:37.908 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:37.908+0000 7fead791f040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-18T00:57:37.950 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:37.949+0000 7fead791f040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-18T00:57:38.059 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:38.058+0000 7fead791f040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-18T00:57:38.064 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T00:57:38.063+0000 7f80e6a1e640 -1 Falling back to public interface 2024-09-18T00:57:38.067 INFO:tasks.ceph.cluster2.osd.1.smithi150.stderr:2024-09-18T00:57:38.066+0000 7f3c01039640 -1 Falling back to public interface 2024-09-18T00:57:38.109 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:38.109+0000 7fead791f040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-18T00:57:38.150 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:38.149+0000 7fead791f040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-18T00:57:38.260 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:38.259+0000 7fead791f040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-18T00:57:38.307 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:38.306+0000 7fead791f040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-18T00:57:38.354 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:38.353+0000 7fead791f040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-18T00:57:38.405 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:38.405+0000 7fead791f040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-18T00:57:38.447 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:38.446+0000 7fead791f040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-18T00:57:38.487 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:38.487+0000 7fead791f040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-18T00:57:39.043 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:39.041+0000 7fead791f040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-18T00:57:40.104 INFO:tasks.ceph.cluster2.mgr.x.smithi078.stderr:2024-09-18T00:57:40.102+0000 7fea77915640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-18T00:57:41.383 INFO:tasks.ceph.cluster2.osd.0.smithi150.stderr:2024-09-18T00:57:41.382+0000 7f8991a0d640 -1 osd.0 0 log_to_monitors true 2024-09-18T00:57:41.819 INFO:tasks.ceph.cluster2.osd.1.smithi150.stderr:2024-09-18T00:57:41.818+0000 7f3c01039640 -1 osd.1 0 log_to_monitors true 2024-09-18T00:57:42.358 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T00:57:42.358+0000 7f80e6a1e640 -1 osd.2 0 log_to_monitors true 2024-09-18T00:57:43.574 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-18T00:57:43.949 INFO:teuthology.misc.health.smithi150.stdout: 2024-09-18T00:57:43.949 INFO:teuthology.misc.health.smithi150.stdout:{"epoch":7,"fsid":"134b3e88-b82a-4f51-8a00-69fd22a05e09","created":"2024-09-18T00:57:35.765780+0000","modified":"2024-09-18T00:57:43.110013+0000","last_up_change":"0.000000","last_in_change":"2024-09-18T00:57:36.604173+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"dda8f346-bbdb-42fd-876f-c6cdec5d1826","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"32875857-ffe6-4037-be43-509984a1afdf","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"0095f5ec-4942-4407-a217-6923831ae2ae","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:57:43.961 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-09-18T00:57:44.011 INFO:tasks.ceph.cluster2.osd.0.smithi150.stderr:2024-09-18T00:57:44.011+0000 7f898db32640 -1 osd.0 0 waiting for initial osdmap 2024-09-18T00:57:44.025 INFO:tasks.ceph.cluster2.osd.1.smithi150.stderr:2024-09-18T00:57:44.024+0000 7f3bfcfc8640 -1 osd.1 0 waiting for initial osdmap 2024-09-18T00:57:44.028 INFO:tasks.ceph.cluster2.osd.0.smithi150.stderr:2024-09-18T00:57:44.028+0000 7f898513e640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T00:57:44.036 INFO:tasks.ceph.cluster2.osd.1.smithi150.stderr:2024-09-18T00:57:44.036+0000 7f3bf4de8640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T00:57:44.864 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T00:57:44.864+0000 7f80e240c640 -1 osd.2 0 waiting for initial osdmap 2024-09-18T00:57:44.874 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T00:57:44.874+0000 7f80da22c640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-18T00:57:50.265 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-09-18T00:57:50.546 INFO:teuthology.misc.health.smithi150.stdout: 2024-09-18T00:57:50.546 INFO:teuthology.misc.health.smithi150.stdout:{"epoch":12,"fsid":"134b3e88-b82a-4f51-8a00-69fd22a05e09","created":"2024-09-18T00:57:35.765780+0000","modified":"2024-09-18T00:57:48.131885+0000","last_up_change":"2024-09-18T00:57:45.120993+0000","last_in_change":"2024-09-18T00:57:36.604173+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-18T00:57:45.280415+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"dda8f346-bbdb-42fd-876f-c6cdec5d1826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6801","nonce":204393861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6803","nonce":204393861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6807","nonce":204393861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6805","nonce":204393861}]},"public_addr":"172.21.15.150:6801/204393861","cluster_addr":"172.21.15.150:6803/204393861","heartbeat_back_addr":"172.21.15.150:6807/204393861","heartbeat_front_addr":"172.21.15.150:6805/204393861","state":["exists","up"]},{"osd":1,"uuid":"32875857-ffe6-4037-be43-509984a1afdf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6817","nonce":1827386980}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6819","nonce":1827386980}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6823","nonce":1827386980}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6821","nonce":1827386980}]},"public_addr":"172.21.15.150:6817/1827386980","cluster_addr":"172.21.15.150:6819/1827386980","heartbeat_back_addr":"172.21.15.150:6823/1827386980","heartbeat_front_addr":"172.21.15.150:6821/1827386980","state":["exists","up"]},{"osd":2,"uuid":"0095f5ec-4942-4407-a217-6923831ae2ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6809","nonce":3623182795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6811","nonce":3623182795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6815","nonce":3623182795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6813","nonce":3623182795}]},"public_addr":"172.21.15.150:6809/3623182795","cluster_addr":"172.21.15.150:6811/3623182795","heartbeat_back_addr":"172.21.15.150:6815/3623182795","heartbeat_front_addr":"172.21.15.150:6813/3623182795","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:42.398707+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:42.782487+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:43.343405+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:57:50.558 DEBUG:teuthology.misc:3 of 3 OSDs are up 2024-09-18T00:57:50.558 INFO:tasks.ceph:Creating RBD pool 2024-09-18T00:57:50.558 DEBUG:teuthology.orchestra.run.smithi150:> sudo ceph --cluster cluster2 osd pool create rbd 8 2024-09-18T00:57:51.146 INFO:teuthology.orchestra.run.smithi150.stderr:pool 'rbd' created 2024-09-18T00:57:51.161 DEBUG:teuthology.orchestra.run.smithi150:> rbd --cluster cluster2 pool init rbd 2024-09-18T00:57:54.177 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2024-09-18T00:57:54.178 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2024-09-18T00:57:54.178 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2024-09-18T00:57:54.178 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2024-09-18T00:57:54.178 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-18T00:57:54.509 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:54.521 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":5,"flags":0,"active_gid":4098,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":216185110},{"type":"v1","addr":"172.21.15.78:6827","nonce":216185110}]},"active_addr":"172.21.15.78:6827/216185110","active_change":"2024-09-18T00:57:39.044268+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":505833581}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":3356279696}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2173028652}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2451878286}]}]} 2024-09-18T00:57:54.523 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2024-09-18T00:57:54.523 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2024-09-18T00:57:54.523 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-18T00:57:54.798 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:54.798 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":16,"fsid":"134b3e88-b82a-4f51-8a00-69fd22a05e09","created":"2024-09-18T00:57:35.765780+0000","modified":"2024-09-18T00:57:54.161985+0000","last_up_change":"2024-09-18T00:57:45.120993+0000","last_in_change":"2024-09-18T00:57:36.604173+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-18T00:57:45.280415+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-18T00:57:50.851259+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"dda8f346-bbdb-42fd-876f-c6cdec5d1826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6801","nonce":204393861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6803","nonce":204393861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6807","nonce":204393861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6805","nonce":204393861}]},"public_addr":"172.21.15.150:6801/204393861","cluster_addr":"172.21.15.150:6803/204393861","heartbeat_back_addr":"172.21.15.150:6807/204393861","heartbeat_front_addr":"172.21.15.150:6805/204393861","state":["exists","up"]},{"osd":1,"uuid":"32875857-ffe6-4037-be43-509984a1afdf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6817","nonce":1827386980}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6819","nonce":1827386980}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6823","nonce":1827386980}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6821","nonce":1827386980}]},"public_addr":"172.21.15.150:6817/1827386980","cluster_addr":"172.21.15.150:6819/1827386980","heartbeat_back_addr":"172.21.15.150:6823/1827386980","heartbeat_front_addr":"172.21.15.150:6821/1827386980","state":["exists","up"]},{"osd":2,"uuid":"0095f5ec-4942-4407-a217-6923831ae2ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6809","nonce":3623182795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6811","nonce":3623182795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6815","nonce":3623182795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6813","nonce":3623182795}]},"public_addr":"172.21.15.150:6809/3623182795","cluster_addr":"172.21.15.150:6811/3623182795","heartbeat_back_addr":"172.21.15.150:6815/3623182795","heartbeat_front_addr":"172.21.15.150:6813/3623182795","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:42.398707+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:42.782487+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:43.343405+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:57:54.811 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2024-09-18T00:57:54.811 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-18T00:57:55.087 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:55.088 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":16,"fsid":"134b3e88-b82a-4f51-8a00-69fd22a05e09","created":"2024-09-18T00:57:35.765780+0000","modified":"2024-09-18T00:57:54.161985+0000","last_up_change":"2024-09-18T00:57:45.120993+0000","last_in_change":"2024-09-18T00:57:36.604173+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-18T00:57:45.280415+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-18T00:57:50.851259+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"dda8f346-bbdb-42fd-876f-c6cdec5d1826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6801","nonce":204393861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6803","nonce":204393861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6807","nonce":204393861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6805","nonce":204393861}]},"public_addr":"172.21.15.150:6801/204393861","cluster_addr":"172.21.15.150:6803/204393861","heartbeat_back_addr":"172.21.15.150:6807/204393861","heartbeat_front_addr":"172.21.15.150:6805/204393861","state":["exists","up"]},{"osd":1,"uuid":"32875857-ffe6-4037-be43-509984a1afdf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6817","nonce":1827386980}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6819","nonce":1827386980}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6823","nonce":1827386980}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6821","nonce":1827386980}]},"public_addr":"172.21.15.150:6817/1827386980","cluster_addr":"172.21.15.150:6819/1827386980","heartbeat_back_addr":"172.21.15.150:6823/1827386980","heartbeat_front_addr":"172.21.15.150:6821/1827386980","state":["exists","up"]},{"osd":2,"uuid":"0095f5ec-4942-4407-a217-6923831ae2ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6809","nonce":3623182795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6811","nonce":3623182795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6815","nonce":3623182795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6813","nonce":3623182795}]},"public_addr":"172.21.15.150:6809/3623182795","cluster_addr":"172.21.15.150:6811/3623182795","heartbeat_back_addr":"172.21.15.150:6815/3623182795","heartbeat_front_addr":"172.21.15.150:6813/3623182795","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:42.398707+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:42.782487+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:43.343405+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T00:57:55.100 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2024-09-18T00:57:55.101 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2024-09-18T00:57:55.101 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2024-09-18T00:57:55.241 INFO:teuthology.orchestra.run.smithi150.stdout:38654705669 2024-09-18T00:57:55.241 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-09-18T00:57:55.244 INFO:teuthology.orchestra.run.smithi150.stdout:34359738373 2024-09-18T00:57:55.244 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-09-18T00:57:55.245 INFO:teuthology.orchestra.run.smithi150.stdout:34359738373 2024-09-18T00:57:55.245 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-09-18T00:57:55.589 INFO:teuthology.orchestra.run.smithi150.stdout:34359738371 2024-09-18T00:57:55.598 INFO:teuthology.orchestra.run.smithi150.stdout:38654705668 2024-09-18T00:57:55.602 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738371 for osd.1 2024-09-18T00:57:55.611 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705668 for osd.2 2024-09-18T00:57:55.626 INFO:teuthology.orchestra.run.smithi150.stdout:34359738371 2024-09-18T00:57:55.639 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738371 for osd.0 2024-09-18T00:57:56.603 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-09-18T00:57:56.613 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-09-18T00:57:56.641 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-09-18T00:57:56.911 INFO:teuthology.orchestra.run.smithi150.stdout:34359738371 2024-09-18T00:57:56.924 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738371 for osd.1 2024-09-18T00:57:56.943 INFO:teuthology.orchestra.run.smithi150.stdout:34359738371 2024-09-18T00:57:56.947 INFO:teuthology.orchestra.run.smithi150.stdout:38654705668 2024-09-18T00:57:56.960 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738371 for osd.0 2024-09-18T00:57:56.963 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705668 for osd.2 2024-09-18T00:57:57.925 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-09-18T00:57:57.961 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-09-18T00:57:57.964 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-09-18T00:57:58.238 INFO:teuthology.orchestra.run.smithi150.stdout:34359738373 2024-09-18T00:57:58.251 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738373 for osd.1 2024-09-18T00:57:58.251 DEBUG:teuthology.parallel:result is None 2024-09-18T00:57:58.295 INFO:teuthology.orchestra.run.smithi150.stdout:34359738373 2024-09-18T00:57:58.303 INFO:teuthology.orchestra.run.smithi150.stdout:38654705670 2024-09-18T00:57:58.308 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738373 for osd.0 2024-09-18T00:57:58.309 DEBUG:teuthology.parallel:result is None 2024-09-18T00:57:58.316 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705670 for osd.2 2024-09-18T00:57:58.316 DEBUG:teuthology.parallel:result is None 2024-09-18T00:57:58.317 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2024-09-18T00:57:58.317 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T00:57:58.629 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:58.629 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-09-18T00:57:58.641 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2024-09-18T00:57:57.050940+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81620,"kb_used_data":528,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283033900,"statfs":{"total":289910292480,"available":289826713600,"internally_reserved":0,"allocated":540672,"data_stored":1304053,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4789,"internal_metadata":82963787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.901791"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.172397+0000","last_change":"2024-09-18T00:57:54.172397+0000","last_active":"2024-09-18T00:57:54.172397+0000","last_peered":"2024-09-18T00:57:54.172397+0000","last_clean":"2024-09-18T00:57:54.172397+0000","last_became_active":"2024-09-18T00:57:52.161899+0000","last_became_peered":"2024-09-18T00:57:52.161899+0000","last_unstale":"2024-09-18T00:57:54.172397+0000","last_undegraded":"2024-09-18T00:57:54.172397+0000","last_fullsized":"2024-09-18T00:57:54.172397+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:00:53.878037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075298400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.172095+0000","last_change":"2024-09-18T00:57:54.172095+0000","last_active":"2024-09-18T00:57:54.172095+0000","last_peered":"2024-09-18T00:57:54.172095+0000","last_clean":"2024-09-18T00:57:54.172095+0000","last_became_active":"2024-09-18T00:57:52.159679+0000","last_became_peered":"2024-09-18T00:57:52.159679+0000","last_unstale":"2024-09-18T00:57:54.172095+0000","last_undegraded":"2024-09-18T00:57:54.172095+0000","last_fullsized":"2024-09-18T00:57:54.172095+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:43:58.913774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054889900000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.172691+0000","last_change":"2024-09-18T00:57:54.172691+0000","last_active":"2024-09-18T00:57:54.172691+0000","last_peered":"2024-09-18T00:57:54.172691+0000","last_clean":"2024-09-18T00:57:54.172691+0000","last_became_active":"2024-09-18T00:57:52.160197+0000","last_became_peered":"2024-09-18T00:57:52.160197+0000","last_unstale":"2024-09-18T00:57:54.172691+0000","last_undegraded":"2024-09-18T00:57:54.172691+0000","last_fullsized":"2024-09-18T00:57:54.172691+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:26:56.936297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096762200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.172438+0000","last_change":"2024-09-18T00:57:54.172438+0000","last_active":"2024-09-18T00:57:54.172438+0000","last_peered":"2024-09-18T00:57:54.172438+0000","last_clean":"2024-09-18T00:57:54.172438+0000","last_became_active":"2024-09-18T00:57:52.159800+0000","last_became_peered":"2024-09-18T00:57:52.159800+0000","last_unstale":"2024-09-18T00:57:54.172438+0000","last_undegraded":"2024-09-18T00:57:54.172438+0000","last_fullsized":"2024-09-18T00:57:54.172438+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:09:57.052840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00074951200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.170987+0000","last_change":"2024-09-18T00:57:54.170233+0000","last_active":"2024-09-18T00:57:54.170987+0000","last_peered":"2024-09-18T00:57:54.170987+0000","last_clean":"2024-09-18T00:57:54.170987+0000","last_became_active":"2024-09-18T00:57:52.159283+0000","last_became_peered":"2024-09-18T00:57:52.159283+0000","last_unstale":"2024-09-18T00:57:54.170987+0000","last_undegraded":"2024-09-18T00:57:54.170987+0000","last_fullsized":"2024-09-18T00:57:54.170987+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:56:30.005098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00113901,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.167662+0000","last_change":"2024-09-18T00:57:54.167662+0000","last_active":"2024-09-18T00:57:54.167662+0000","last_peered":"2024-09-18T00:57:54.167662+0000","last_clean":"2024-09-18T00:57:54.167662+0000","last_became_active":"2024-09-18T00:57:52.160105+0000","last_became_peered":"2024-09-18T00:57:52.160105+0000","last_unstale":"2024-09-18T00:57:54.167662+0000","last_undegraded":"2024-09-18T00:57:54.167662+0000","last_fullsized":"2024-09-18T00:57:54.167662+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:10:35.692324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041490800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.167698+0000","last_change":"2024-09-18T00:57:54.167698+0000","last_active":"2024-09-18T00:57:54.167698+0000","last_peered":"2024-09-18T00:57:54.167698+0000","last_clean":"2024-09-18T00:57:54.167698+0000","last_became_active":"2024-09-18T00:57:52.159792+0000","last_became_peered":"2024-09-18T00:57:52.159792+0000","last_unstale":"2024-09-18T00:57:54.167698+0000","last_undegraded":"2024-09-18T00:57:54.167698+0000","last_fullsized":"2024-09-18T00:57:54.167698+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:42:59.227183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045536600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.172073+0000","last_change":"2024-09-18T00:57:54.172073+0000","last_active":"2024-09-18T00:57:54.172073+0000","last_peered":"2024-09-18T00:57:54.172073+0000","last_clean":"2024-09-18T00:57:54.172073+0000","last_became_active":"2024-09-18T00:57:52.158758+0000","last_became_peered":"2024-09-18T00:57:52.158758+0000","last_unstale":"2024-09-18T00:57:54.172073+0000","last_undegraded":"2024-09-18T00:57:54.172073+0000","last_fullsized":"2024-09-18T00:57:54.172073+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:44:25.148787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053750599999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'97","reported_seq":147,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.171569+0000","last_change":"2024-09-18T00:57:47.137803+0000","last_active":"2024-09-18T00:57:54.171569+0000","last_peered":"2024-09-18T00:57:54.171569+0000","last_clean":"2024-09-18T00:57:54.171569+0000","last_became_active":"2024-09-18T00:57:47.136874+0000","last_became_peered":"2024-09-18T00:57:47.136874+0000","last_unstale":"2024-09-18T00:57:54.171569+0000","last_undegraded":"2024-09-18T00:57:54.171569+0000","last_fullsized":"2024-09-18T00:57:54.171569+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_clean_scrub_stamp":"2024-09-18T00:57:46.126096+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T04:09:27.863635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705670,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27164,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344676,"statfs":{"total":96636764160,"available":96608948224,"internally_reserved":0,"allocated":139264,"data_stored":41093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27224,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344616,"statfs":{"total":96636764160,"available":96608886784,"internally_reserved":0,"allocated":200704,"data_stored":631480,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":200704,"data_stored":631480,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1599,"internal_metadata":27654593},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T00:57:58.642 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T00:57:58.915 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:58.915 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-09-18T00:57:58.927 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2024-09-18T00:57:57.050940+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81620,"kb_used_data":528,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283033900,"statfs":{"total":289910292480,"available":289826713600,"internally_reserved":0,"allocated":540672,"data_stored":1304053,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4789,"internal_metadata":82963787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.901791"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.172397+0000","last_change":"2024-09-18T00:57:54.172397+0000","last_active":"2024-09-18T00:57:54.172397+0000","last_peered":"2024-09-18T00:57:54.172397+0000","last_clean":"2024-09-18T00:57:54.172397+0000","last_became_active":"2024-09-18T00:57:52.161899+0000","last_became_peered":"2024-09-18T00:57:52.161899+0000","last_unstale":"2024-09-18T00:57:54.172397+0000","last_undegraded":"2024-09-18T00:57:54.172397+0000","last_fullsized":"2024-09-18T00:57:54.172397+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:00:53.878037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075298400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.172095+0000","last_change":"2024-09-18T00:57:54.172095+0000","last_active":"2024-09-18T00:57:54.172095+0000","last_peered":"2024-09-18T00:57:54.172095+0000","last_clean":"2024-09-18T00:57:54.172095+0000","last_became_active":"2024-09-18T00:57:52.159679+0000","last_became_peered":"2024-09-18T00:57:52.159679+0000","last_unstale":"2024-09-18T00:57:54.172095+0000","last_undegraded":"2024-09-18T00:57:54.172095+0000","last_fullsized":"2024-09-18T00:57:54.172095+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:43:58.913774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054889900000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.172691+0000","last_change":"2024-09-18T00:57:54.172691+0000","last_active":"2024-09-18T00:57:54.172691+0000","last_peered":"2024-09-18T00:57:54.172691+0000","last_clean":"2024-09-18T00:57:54.172691+0000","last_became_active":"2024-09-18T00:57:52.160197+0000","last_became_peered":"2024-09-18T00:57:52.160197+0000","last_unstale":"2024-09-18T00:57:54.172691+0000","last_undegraded":"2024-09-18T00:57:54.172691+0000","last_fullsized":"2024-09-18T00:57:54.172691+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:26:56.936297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096762200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.172438+0000","last_change":"2024-09-18T00:57:54.172438+0000","last_active":"2024-09-18T00:57:54.172438+0000","last_peered":"2024-09-18T00:57:54.172438+0000","last_clean":"2024-09-18T00:57:54.172438+0000","last_became_active":"2024-09-18T00:57:52.159800+0000","last_became_peered":"2024-09-18T00:57:52.159800+0000","last_unstale":"2024-09-18T00:57:54.172438+0000","last_undegraded":"2024-09-18T00:57:54.172438+0000","last_fullsized":"2024-09-18T00:57:54.172438+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:09:57.052840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00074951200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.170987+0000","last_change":"2024-09-18T00:57:54.170233+0000","last_active":"2024-09-18T00:57:54.170987+0000","last_peered":"2024-09-18T00:57:54.170987+0000","last_clean":"2024-09-18T00:57:54.170987+0000","last_became_active":"2024-09-18T00:57:52.159283+0000","last_became_peered":"2024-09-18T00:57:52.159283+0000","last_unstale":"2024-09-18T00:57:54.170987+0000","last_undegraded":"2024-09-18T00:57:54.170987+0000","last_fullsized":"2024-09-18T00:57:54.170987+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:56:30.005098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00113901,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.167662+0000","last_change":"2024-09-18T00:57:54.167662+0000","last_active":"2024-09-18T00:57:54.167662+0000","last_peered":"2024-09-18T00:57:54.167662+0000","last_clean":"2024-09-18T00:57:54.167662+0000","last_became_active":"2024-09-18T00:57:52.160105+0000","last_became_peered":"2024-09-18T00:57:52.160105+0000","last_unstale":"2024-09-18T00:57:54.167662+0000","last_undegraded":"2024-09-18T00:57:54.167662+0000","last_fullsized":"2024-09-18T00:57:54.167662+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:10:35.692324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041490800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.167698+0000","last_change":"2024-09-18T00:57:54.167698+0000","last_active":"2024-09-18T00:57:54.167698+0000","last_peered":"2024-09-18T00:57:54.167698+0000","last_clean":"2024-09-18T00:57:54.167698+0000","last_became_active":"2024-09-18T00:57:52.159792+0000","last_became_peered":"2024-09-18T00:57:52.159792+0000","last_unstale":"2024-09-18T00:57:54.167698+0000","last_undegraded":"2024-09-18T00:57:54.167698+0000","last_fullsized":"2024-09-18T00:57:54.167698+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:42:59.227183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045536600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.172073+0000","last_change":"2024-09-18T00:57:54.172073+0000","last_active":"2024-09-18T00:57:54.172073+0000","last_peered":"2024-09-18T00:57:54.172073+0000","last_clean":"2024-09-18T00:57:54.172073+0000","last_became_active":"2024-09-18T00:57:52.158758+0000","last_became_peered":"2024-09-18T00:57:52.158758+0000","last_unstale":"2024-09-18T00:57:54.172073+0000","last_undegraded":"2024-09-18T00:57:54.172073+0000","last_fullsized":"2024-09-18T00:57:54.172073+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:44:25.148787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053750599999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'97","reported_seq":147,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-18T00:57:54.171569+0000","last_change":"2024-09-18T00:57:47.137803+0000","last_active":"2024-09-18T00:57:54.171569+0000","last_peered":"2024-09-18T00:57:54.171569+0000","last_clean":"2024-09-18T00:57:54.171569+0000","last_became_active":"2024-09-18T00:57:47.136874+0000","last_became_peered":"2024-09-18T00:57:47.136874+0000","last_unstale":"2024-09-18T00:57:54.171569+0000","last_undegraded":"2024-09-18T00:57:54.171569+0000","last_fullsized":"2024-09-18T00:57:54.171569+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_clean_scrub_stamp":"2024-09-18T00:57:46.126096+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T04:09:27.863635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705670,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27164,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344676,"statfs":{"total":96636764160,"available":96608948224,"internally_reserved":0,"allocated":139264,"data_stored":41093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27224,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344616,"statfs":{"total":96636764160,"available":96608886784,"internally_reserved":0,"allocated":200704,"data_stored":631480,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27232,"kb_used_data":196,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344608,"statfs":{"total":96636764160,"available":96608878592,"internally_reserved":0,"allocated":200704,"data_stored":631480,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1599,"internal_metadata":27654593},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T00:57:58.928 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2024-09-18T00:57:58.928 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2024-09-18T00:57:58.928 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2024-09-18T00:57:58.928 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2024-09-18T00:57:59.237 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T00:57:59.237 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-18T00:57:59.250 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2024-09-18T00:57:59.250 INFO:teuthology.run_tasks:Running task exec... 2024-09-18T00:57:59.264 INFO:teuthology.task.exec:Executing custom commands... 2024-09-18T00:57:59.264 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@smithi150.front.sepia.ceph.com 2024-09-18T00:57:59.265 DEBUG:teuthology.orchestra.run.smithi150:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2024-09-18T00:57:59.616 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror] 2024-09-18T00:57:59.617 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBKJepmlWikOBAA4RxHqKWSrV7yNRNnrCpYSA== 2024-09-18T00:57:59.617 INFO:teuthology.orchestra.run.smithi150.stdout: caps mon = "profile rbd-mirror-peer" 2024-09-18T00:57:59.617 INFO:teuthology.orchestra.run.smithi150.stdout: caps osd = "profile rbd" 2024-09-18T00:57:59.617 INFO:teuthology.orchestra.run.smithi150.stderr:updated caps for client.mirror 2024-09-18T00:57:59.629 DEBUG:teuthology.orchestra.run.smithi150:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-18T00:57:59.979 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.0] 2024-09-18T00:57:59.979 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBLJepm+gV2ABAAY7eST+q2bk/LCtoPp70xtQ== 2024-09-18T00:57:59.979 INFO:teuthology.orchestra.run.smithi150.stdout: caps mon = "profile rbd-mirror" 2024-09-18T00:57:59.979 INFO:teuthology.orchestra.run.smithi150.stdout: caps osd = "profile rbd" 2024-09-18T00:57:59.979 INFO:teuthology.orchestra.run.smithi150.stderr:updated caps for client.mirror.0 2024-09-18T00:57:59.992 DEBUG:teuthology.orchestra.run.smithi150:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-18T00:58:00.346 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.1] 2024-09-18T00:58:00.346 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBLJepm2YK7AxAAC6u3Le1LVG1gXYw+1KMdaA== 2024-09-18T00:58:00.347 INFO:teuthology.orchestra.run.smithi150.stdout: caps mon = "profile rbd-mirror" 2024-09-18T00:58:00.347 INFO:teuthology.orchestra.run.smithi150.stdout: caps osd = "profile rbd" 2024-09-18T00:58:00.347 INFO:teuthology.orchestra.run.smithi150.stderr:updated caps for client.mirror.1 2024-09-18T00:58:00.360 DEBUG:teuthology.orchestra.run.smithi150:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-18T00:58:00.709 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.2] 2024-09-18T00:58:00.709 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBLJepmNutZCRAA6JtUbX/wfLPgRXSh7RVjIg== 2024-09-18T00:58:00.710 INFO:teuthology.orchestra.run.smithi150.stdout: caps mon = "profile rbd-mirror" 2024-09-18T00:58:00.710 INFO:teuthology.orchestra.run.smithi150.stdout: caps osd = "profile rbd" 2024-09-18T00:58:00.710 INFO:teuthology.orchestra.run.smithi150.stderr:updated caps for client.mirror.2 2024-09-18T00:58:00.722 DEBUG:teuthology.orchestra.run.smithi150:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-18T00:58:01.073 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.3] 2024-09-18T00:58:01.073 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBLJepmfB3tDhAA7712lnLOrPrs6c5NspeRXw== 2024-09-18T00:58:01.073 INFO:teuthology.orchestra.run.smithi150.stdout: caps mon = "profile rbd-mirror" 2024-09-18T00:58:01.073 INFO:teuthology.orchestra.run.smithi150.stdout: caps osd = "profile rbd" 2024-09-18T00:58:01.073 INFO:teuthology.orchestra.run.smithi150.stderr:updated caps for client.mirror.3 2024-09-18T00:58:01.086 DEBUG:teuthology.orchestra.run.smithi150:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2024-09-18T00:58:01.433 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror] 2024-09-18T00:58:01.433 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmdGA2GhAAgyv5NSZlHf9LaVXZ4UDltw== 2024-09-18T00:58:01.434 INFO:teuthology.orchestra.run.smithi150.stdout: caps mon = "profile rbd-mirror-peer" 2024-09-18T00:58:01.434 INFO:teuthology.orchestra.run.smithi150.stdout: caps osd = "profile rbd" 2024-09-18T00:58:01.434 INFO:teuthology.orchestra.run.smithi150.stderr:updated caps for client.mirror 2024-09-18T00:58:01.447 DEBUG:teuthology.orchestra.run.smithi150:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-18T00:58:01.796 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.0] 2024-09-18T00:58:01.796 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmFz/MHxAAqJu6cWplUA4Mjmd6pkoKLA== 2024-09-18T00:58:01.796 INFO:teuthology.orchestra.run.smithi150.stdout: caps mon = "profile rbd-mirror" 2024-09-18T00:58:01.796 INFO:teuthology.orchestra.run.smithi150.stdout: caps osd = "profile rbd" 2024-09-18T00:58:01.796 INFO:teuthology.orchestra.run.smithi150.stderr:updated caps for client.mirror.0 2024-09-18T00:58:01.809 DEBUG:teuthology.orchestra.run.smithi150:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-18T00:58:02.158 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.1] 2024-09-18T00:58:02.158 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmuY5lJRAAZTS7qN1Weg4c8TXjPtOVLw== 2024-09-18T00:58:02.158 INFO:teuthology.orchestra.run.smithi150.stdout: caps mon = "profile rbd-mirror" 2024-09-18T00:58:02.158 INFO:teuthology.orchestra.run.smithi150.stdout: caps osd = "profile rbd" 2024-09-18T00:58:02.158 INFO:teuthology.orchestra.run.smithi150.stderr:updated caps for client.mirror.1 2024-09-18T00:58:02.171 DEBUG:teuthology.orchestra.run.smithi150:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-18T00:58:02.518 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.2] 2024-09-18T00:58:02.519 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmbDgAKxAA6bfCvEaFmN8fGnAdi1k/Eg== 2024-09-18T00:58:02.519 INFO:teuthology.orchestra.run.smithi150.stdout: caps mon = "profile rbd-mirror" 2024-09-18T00:58:02.519 INFO:teuthology.orchestra.run.smithi150.stdout: caps osd = "profile rbd" 2024-09-18T00:58:02.519 INFO:teuthology.orchestra.run.smithi150.stderr:updated caps for client.mirror.2 2024-09-18T00:58:02.532 DEBUG:teuthology.orchestra.run.smithi150:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-09-18T00:58:02.886 INFO:teuthology.orchestra.run.smithi150.stdout:[client.mirror.3] 2024-09-18T00:58:02.886 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQBxJepmgUGPMBAAaRcvmTZPA8XI8vsyRNxtcQ== 2024-09-18T00:58:02.886 INFO:teuthology.orchestra.run.smithi150.stdout: caps mon = "profile rbd-mirror" 2024-09-18T00:58:02.886 INFO:teuthology.orchestra.run.smithi150.stdout: caps osd = "profile rbd" 2024-09-18T00:58:02.886 INFO:teuthology.orchestra.run.smithi150.stderr:updated caps for client.mirror.3 2024-09-18T00:58:02.897 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-18T00:58:02.907 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-18T00:58:02.908 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-18T00:58:02.910 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-18T00:58:02.910 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-18T00:58:02.936 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-18T00:58:02.936 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-18T00:58:02.938 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-18T00:58:02.938 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-18T00:58:02.970 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-18T00:58:02.970 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-18T00:58:02.972 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-18T00:58:02.973 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-18T00:58:03.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-09-18T00:58:03.005 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-09-18T00:58:03.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-09-18T00:58:03.007 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-18T00:58:03.045 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-18T00:58:03.045 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-18T00:58:03.047 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-18T00:58:03.047 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-18T00:58:03.079 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-18T00:58:03.079 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-18T00:58:03.081 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-18T00:58:03.082 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-18T00:58:03.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-18T00:58:03.112 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-18T00:58:03.114 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-18T00:58:03.114 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-09-18T00:58:03.146 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2024-09-18T00:58:03.146 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2024-09-18T00:58:03.148 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2024-09-18T00:58:03.148 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2024-09-18T00:58:03.180 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1726621083 2024-09-18T00:58:03.180 DEBUG:tasks.rbd_mirror_thrash:Yielding 2024-09-18T00:58:03.180 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2024-09-18T00:58:03.212 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1726621083 2024-09-18T00:58:03.212 DEBUG:tasks.rbd_mirror_thrash:Yielding 2024-09-18T00:58:03.212 INFO:teuthology.run_tasks:Running task workunit... 2024-09-18T00:58:03.249 INFO:tasks.workunit:Pulling workunits from ref b0d356b98e08f6bc59ef15652f0d0e4298fc83c7 2024-09-18T00:58:03.250 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-18T00:58:03.250 DEBUG:teuthology.orchestra.run.smithi150:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-18T00:58:03.251 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1 2024-09-18T00:58:03.251 INFO:tasks.rbd_mirror_thrash:waiting for 88 secs before thrashing 2024-09-18T00:58:03.251 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2 2024-09-18T00:58:03.251 INFO:tasks.rbd_mirror_thrash:waiting for 81 secs before thrashing 2024-09-18T00:58:03.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T00:58:03.266 INFO:teuthology.orchestra.run.smithi150.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2024-09-18T00:58:03.266 DEBUG:teuthology.orchestra.run.smithi150:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-18T00:58:03.322 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-18T00:58:03.322 DEBUG:teuthology.orchestra.run.smithi150:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2024-09-18T00:58:03.377 INFO:tasks.workunit:timeout=6h 2024-09-18T00:58:03.378 INFO:tasks.workunit:cleanup=True 2024-09-18T00:58:03.378 DEBUG:teuthology.orchestra.run.smithi150:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout b0d356b98e08f6bc59ef15652f0d0e4298fc83c7 2024-09-18T00:58:03.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2024-09-18T00:58:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Note: switching to 'b0d356b98e08f6bc59ef15652f0d0e4298fc83c7'. 2024-09-18T00:58:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T00:58:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-09-18T00:58:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:changes and commit them, and you can discard any commits you make in this 2024-09-18T00:58:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:state without impacting any branches by switching back to a branch. 2024-09-18T00:58:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T00:58:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:If you want to create a new branch to retain commits you create, you may 2024-09-18T00:58:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:do so (now or later) by using -c with the switch command. Example: 2024-09-18T00:58:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T00:58:59.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: git switch -c 2024-09-18T00:58:59.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T00:58:59.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Or undo this operation with: 2024-09-18T00:58:59.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T00:58:59.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: git switch - 2024-09-18T00:58:59.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T00:58:59.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-09-18T00:58:59.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T00:58:59.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:HEAD is now at b0d356b98e0 Merge pull request #59811 from afreen23/wip-68079-squid 2024-09-18T00:58:59.443 DEBUG:teuthology.orchestra.run.smithi150:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2024-09-18T00:58:59.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-18T00:58:59.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2024-09-18T00:58:59.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-18T00:59:00.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-18T00:59:00.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-18T00:59:00.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2024-09-18T00:59:00.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2024-09-18T00:59:00.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-18T00:59:00.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2024-09-18T00:59:00.415 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-09-18T00:59:00.415 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2024-09-18T00:59:00.446 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror... 2024-09-18T00:59:00.447 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh... 2024-09-18T00:59:00.447 DEBUG:teuthology.orchestra.run.smithi150:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=b0d356b98e08f6bc59ef15652f0d0e4298fc83c7 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=image RBD_IMAGE_FEATURES=layering,exclusive-lock RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-09-18T00:59:00.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ IMAGE_COUNT=50 2024-09-18T00:59:00.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ export LOCKDEP=0 2024-09-18T00:59:00.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ LOCKDEP=0 2024-09-18T00:59:00.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-09-18T00:59:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2024-09-18T00:59:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ type xmlstarlet 2024-09-18T00:59:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ XMLSTARLET=xmlstarlet 2024-09-18T00:59:00.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ RBD_MIRROR_INSTANCES=4 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CLUSTER1=cluster1 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CLUSTER2=cluster2 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ PEER_CLUSTER_SUFFIX= 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ POOL=mirror 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ PARENT_POOL=mirror_parent 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ NS1=ns1 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ NS2=ns2 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ TEMPDIR= 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ID=mirror 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ RBD_MIRROR_MODE=snapshot 2024-09-18T00:59:00.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ MIRROR_POOL_MODE=image 2024-09-18T00:59:00.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' snapshot = snapshot ']' 2024-09-18T00:59:00.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ MIRROR_POOL_MODE=image 2024-09-18T00:59:00.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ export 'CEPH_ARGS=--id mirror' 2024-09-18T00:59:00.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS='--id mirror' 2024-09-18T00:59:00.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ LAST_MIRROR_INSTANCE=3 2024-09-18T00:59:00.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-09-18T00:59:00.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2024-09-18T00:59:00.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2024-09-18T00:59:00.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_BIN=. 2024-09-18T00:59:00.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_SRC=. 2024-09-18T00:59:00.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' -e CMakeCache.txt ']' 2024-09-18T00:59:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 -gt 0 ']' 2024-09-18T00:59:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ setup 2024-09-18T00:59:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local c 2024-09-18T00:59:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ trap 'cleanup $?' INT TERM EXIT 2024-09-18T00:59:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ setup_tempdir 2024-09-18T00:59:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:00.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mktemp -d 2024-09-18T00:59:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ TEMPDIR=/tmp/tmp.yeNdCEAq6i 2024-09-18T00:59:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -z 1 ']' 2024-09-18T00:59:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ setup_pools cluster1 cluster2 2024-09-18T00:59:00.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T00:59:00.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local remote_cluster=cluster2 2024-09-18T00:59:00.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mon_map_file 2024-09-18T00:59:00.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mon_addr 2024-09-18T00:59:00.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local admin_key_file 2024-09-18T00:59:00.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local uuid 2024-09-18T00:59:00.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T00:59:00.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2024-09-18T00:59:01.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:pool 'mirror' already exists 2024-09-18T00:59:01.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T00:59:01.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2024-09-18T00:59:02.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:pool 'mirror_parent' already exists 2024-09-18T00:59:02.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T00:59:02.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 pool init mirror 2024-09-18T00:59:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T00:59:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2024-09-18T00:59:08.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:08.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T00:59:08.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror image 2024-09-18T00:59:08.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2024-09-18T00:59:08.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2024-09-18T00:59:08.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2024-09-18T00:59:08.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2024-09-18T00:59:08.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2024-09-18T00:59:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -z ']' 2024-09-18T00:59:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -z ']' 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ peer_add cluster1 mirror cluster2 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local client_cluster=cluster2 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local remote_cluster=cluster2 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local uuid_var_name 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local error_code 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local peer_uuid 2024-09-18T00:59:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 16 32 2024-09-18T00:59:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T00:59:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2024-09-18T00:59:09.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ peer_uuid=14bb9619-09e0-4acf-852d-182bba391b00 2024-09-18T00:59:09.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=0 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -eq 17 ']' 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 0 -eq 0 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ peer_add cluster1 mirror_parent cluster2 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror_parent 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local client_cluster=cluster2 2024-09-18T00:59:09.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local remote_cluster=cluster2 2024-09-18T00:59:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local uuid_var_name 2024-09-18T00:59:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local error_code 2024-09-18T00:59:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local peer_uuid 2024-09-18T00:59:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 16 32 2024-09-18T00:59:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T00:59:09.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2024-09-18T00:59:09.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ peer_uuid=c4210fee-cc8b-4129-9a4b-354e43888652 2024-09-18T00:59:09.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=0 2024-09-18T00:59:09.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T00:59:09.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -eq 17 ']' 2024-09-18T00:59:09.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 0 -eq 0 2024-09-18T00:59:09.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:09.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T00:59:09.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ setup_pools cluster2 cluster1 2024-09-18T00:59:09.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T00:59:09.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local remote_cluster=cluster1 2024-09-18T00:59:09.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mon_map_file 2024-09-18T00:59:09.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mon_addr 2024-09-18T00:59:09.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local admin_key_file 2024-09-18T00:59:09.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local uuid 2024-09-18T00:59:09.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T00:59:09.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2024-09-18T00:59:10.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:pool 'mirror' already exists 2024-09-18T00:59:10.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T00:59:10.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2024-09-18T00:59:11.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:pool 'mirror_parent' already exists 2024-09-18T00:59:11.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T00:59:11.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 pool init mirror 2024-09-18T00:59:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T00:59:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2024-09-18T00:59:17.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:17.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T00:59:17.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror image 2024-09-18T00:59:17.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2024-09-18T00:59:17.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2024-09-18T00:59:17.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2024-09-18T00:59:17.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2024-09-18T00:59:17.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2024-09-18T00:59:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -z ']' 2024-09-18T00:59:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -z ']' 2024-09-18T00:59:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ peer_add cluster2 mirror cluster1 2024-09-18T00:59:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T00:59:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:17.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local client_cluster=cluster1 2024-09-18T00:59:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local remote_cluster=cluster1 2024-09-18T00:59:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local uuid_var_name 2024-09-18T00:59:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local error_code 2024-09-18T00:59:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local peer_uuid 2024-09-18T00:59:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 16 32 2024-09-18T00:59:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T00:59:17.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2024-09-18T00:59:17.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ peer_uuid=cfaf647d-5407-47c4-bd50-bf03457c29db 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=0 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -eq 17 ']' 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 0 -eq 0 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ peer_add cluster2 mirror_parent cluster1 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror_parent 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local client_cluster=cluster1 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local remote_cluster=cluster1 2024-09-18T00:59:17.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local uuid_var_name 2024-09-18T00:59:17.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:17.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local error_code 2024-09-18T00:59:17.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local peer_uuid 2024-09-18T00:59:17.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 16 32 2024-09-18T00:59:17.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T00:59:17.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2024-09-18T00:59:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ peer_uuid=14057cfb-32c0-45ce-bbdf-cdf2ffe1efef 2024-09-18T00:59:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=0 2024-09-18T00:59:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T00:59:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -eq 17 ']' 2024-09-18T00:59:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 0 -eq 0 2024-09-18T00:59:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T00:59:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T00:59:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ start_mirrors cluster1 2024-09-18T00:59:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T00:59:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 0 3 2024-09-18T00:59:17.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:17.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ start_mirror cluster1:0 2024-09-18T00:59:17.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1:0 2024-09-18T00:59:17.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance 2024-09-18T00:59:17.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-09-18T00:59:17.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local val=cluster1:0 2024-09-18T00:59:17.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster_var_name=cluster 2024-09-18T00:59:17.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance_var_name=instance 2024-09-18T00:59:17.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cluster=cluster1 2024-09-18T00:59:17.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ instance=0 2024-09-18T00:59:17.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:17.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval cluster=cluster1 2024-09-18T00:59:17.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:17.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval instance=0 2024-09-18T00:59:17.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T00:59:17.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T00:59:17.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T00:59:17.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:17.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ start_mirror cluster1:1 2024-09-18T00:59:17.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1:1 2024-09-18T00:59:17.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local val=cluster1:1 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster_var_name=cluster 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance_var_name=instance 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cluster=cluster1 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ instance=1 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval cluster=cluster1 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval instance=1 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T00:59:17.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ start_mirror cluster1:2 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1:2 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local val=cluster1:2 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster_var_name=cluster 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance_var_name=instance 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cluster=cluster1 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ instance=2 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 2 = cluster1:2 ']' 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval cluster=cluster1 2024-09-18T00:59:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval instance=2 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ start_mirror cluster1:3 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1:3 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local val=cluster1:3 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster_var_name=cluster 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance_var_name=instance 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cluster=cluster1 2024-09-18T00:59:17.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ instance=3 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 3 = cluster1:3 ']' 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval cluster=cluster1 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval instance=3 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ start_mirrors cluster2 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 0 3 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ start_mirror cluster2:0 2024-09-18T00:59:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2:0 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set_cluster_instance cluster2:0 cluster instance 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local val=cluster2:0 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster_var_name=cluster 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance_var_name=instance 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cluster=cluster2 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ instance=0 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 = cluster2:0 ']' 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval cluster=cluster2 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster2 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval instance=0 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T00:59:17.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ start_mirror cluster2:1 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2:1 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set_cluster_instance cluster2:1 cluster instance 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local val=cluster2:1 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster_var_name=cluster 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance_var_name=instance 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cluster=cluster2 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ instance=1 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 1 = cluster2:1 ']' 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval cluster=cluster2 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster2 2024-09-18T00:59:17.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval instance=1 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ start_mirror cluster2:2 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2:2 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set_cluster_instance cluster2:2 cluster instance 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local val=cluster2:2 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster_var_name=cluster 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance_var_name=instance 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cluster=cluster2 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ instance=2 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 2 = cluster2:2 ']' 2024-09-18T00:59:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval cluster=cluster2 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster2 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval instance=2 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ start_mirror cluster2:3 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2:3 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set_cluster_instance cluster2:3 cluster instance 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local val=cluster2:3 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster_var_name=cluster 2024-09-18T00:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance_var_name=instance 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cluster=cluster2 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ instance=3 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 3 = cluster2:3 ']' 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval cluster=cluster2 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster2 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval instance=3 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ testlog 'TEST: add image and test replay after client crashes' 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ tee -a /tmp/tmp.yeNdCEAq6i/rbd-mirror.test.log 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ date '+%F %T' 2024-09-18T00:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo 2024-09-18 00:59:17 TEST: add image and test replay after client crashes 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:2024-09-18 00:59:17 TEST: add image and test replay after client crashes 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=test 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 512M 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror test 512M 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T00:59:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T00:59:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 512M ']' 2024-09-18T00:59:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=512M 2024-09-18T00:59:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T00:59:17.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/test 2024-09-18T00:59:17.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T00:59:17.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:17.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T00:59:17.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T00:59:17.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror test snapshot 2024-09-18T00:59:17.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T00:59:17.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:17.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T00:59:17.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T00:59:17.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2024-09-18T00:59:18.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T00:59:18.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/test 2024-09-18T00:59:18.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T00:59:18.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T00:59:18.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T00:59:18.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T00:59:18.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10ada9229771 2024-09-18T00:59:18.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10ada9229771 2024-09-18T00:59:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T00:59:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T00:59:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T00:59:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T00:59:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:17 2024 2024-09-18T00:59:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 00:59:17 2024 2024-09-18T00:59:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:17 2024 2024-09-18T00:59:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T00:59:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T00:59:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T00:59:18.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T00:59:18.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T00:59:18.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T00:59:18.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:18.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T00:59:18.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T00:59:18.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T00:59:18.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:18.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T00:59:18.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T00:59:18.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T00:59:18.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:18.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T00:59:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T00:59:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T00:59:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T00:59:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T00:59:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T00:59:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T00:59:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T00:59:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T00:59:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T00:59:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T00:59:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T00:59:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T00:59:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T00:59:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T00:59:19.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T00:59:19.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:19.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T00:59:19.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T00:59:19.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T00:59:19.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T00:59:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T00:59:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T00:59:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T00:59:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T00:59:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T00:59:19.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:19.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:19.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:19.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T00:59:19.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:19.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:19.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:19.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:19.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:19.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T00:59:19.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T00:59:19.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T00:59:19.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:19.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:19.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:19.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T00:59:19.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:19.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:19.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-18T00:59:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T00:59:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T00:59:19.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T00:59:19.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T00:59:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-18T00:59:19.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:19.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:19.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:19.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T00:59:19.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T00:59:19.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:19.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:19.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T00:59:19.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T00:59:19.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T00:59:19.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:19.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:19.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:19.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-18T00:59:19.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:19.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:19.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:19.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:19.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 1 2024-09-18T00:59:20.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T00:59:20.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T00:59:20.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:20.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:20.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:20.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:20.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:20.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T00:59:20.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T00:59:20.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T00:59:20.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:20.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:20.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:20.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T00:59:20.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:20.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:20.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T00:59:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T00:59:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T00:59:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T00:59:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:20.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:20.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T00:59:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T00:59:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T00:59:20.797 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:20.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T00:59:20.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:20.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:20.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:20.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:20.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-18T00:59:20.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T00:59:20.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:20.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T00:59:20.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T00:59:20.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T00:59:20.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:20.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:20.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:20.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-18T00:59:20.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:20.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:20.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-18T00:59:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T00:59:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T00:59:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T00:59:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:20.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:20.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T00:59:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T00:59:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T00:59:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T00:59:20.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T00:59:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:20.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:20.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-18T00:59:21.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:21.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:21.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:21.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:21.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 2 2024-09-18T00:59:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T00:59:23.055 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T00:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T00:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T00:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T00:59:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T00:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:23.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T00:59:23.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:23.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:23.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T00:59:23.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:23.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:23.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:23.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:23.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:23.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T00:59:23.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T00:59:23.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T00:59:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T00:59:23.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:23.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:23.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:23.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:23.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-18T00:59:23.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T00:59:23.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:23.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:23.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T00:59:23.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T00:59:23.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:23.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:23.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:23.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:23.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T00:59:23.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T00:59:23.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:23.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:23.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T00:59:23.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T00:59:23.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T00:59:23.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:23.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:23.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:23.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-18T00:59:23.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:23.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:23.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T00:59:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T00:59:23.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T00:59:23.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:23.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:23.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T00:59:23.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T00:59:23.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T00:59:23.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:23.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:23.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:23.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-18T00:59:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 4 2024-09-18T00:59:24.252 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4560727256165156) > weight (0.25) 2024-09-18T00:59:24.252 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.321952228655342) > weight (0.25) 2024-09-18T00:59:24.252 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.4983467606671984) > weight (0.25) 2024-09-18T00:59:24.252 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8899927973867755) > weight (0.25) 2024-09-18T00:59:24.252 INFO:tasks.rbd_mirror_thrash:waiting for 108 secs before thrashing 2024-09-18T00:59:27.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T00:59:27.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T00:59:27.443 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:27.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:27.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:27.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:27.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:27.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:27.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T00:59:27.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T00:59:27.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T00:59:27.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:27.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:27.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:27.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T00:59:27.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:27.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:27.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T00:59:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T00:59:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T00:59:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T00:59:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:27.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T00:59:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T00:59:27.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T00:59:27.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:27.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:27.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:27.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T00:59:27.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:27.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:27.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-18T00:59:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T00:59:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T00:59:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T00:59:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:27.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:27.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T00:59:27.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T00:59:27.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:27.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:27.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T00:59:27.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T00:59:27.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T00:59:27.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:27.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:27.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:27.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-18T00:59:27.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:27.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:27.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:27.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:27.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-18T00:59:27.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T00:59:27.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:27.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:27.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T00:59:27.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T00:59:27.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:27.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:27.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:27.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:27.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T00:59:27.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T00:59:27.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:27.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:27.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T00:59:27.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T00:59:27.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T00:59:27.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:27.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:27.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:27.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-18T00:59:27.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 8 2024-09-18T00:59:31.252 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7420447929738572) > weight (0.25) 2024-09-18T00:59:31.252 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6009744297693275) > weight (0.25) 2024-09-18T00:59:31.253 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.38961377135406705) > weight (0.25) 2024-09-18T00:59:31.253 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7486830591685087) > weight (0.25) 2024-09-18T00:59:31.253 INFO:tasks.rbd_mirror_thrash:waiting for 85 secs before thrashing 2024-09-18T00:59:35.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T00:59:35.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T00:59:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T00:59:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T00:59:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:35.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T00:59:35.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T00:59:35.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T00:59:35.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:35.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:35.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:35.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T00:59:35.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:35.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:35.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:35.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:35.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T00:59:35.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T00:59:35.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:35.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T00:59:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T00:59:35.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T00:59:35.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:35.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:35.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:35.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T00:59:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:36.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-09-18T00:59:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T00:59:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T00:59:36.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:36.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T00:59:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T00:59:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T00:59:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T00:59:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T00:59:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T00:59:36.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T00:59:36.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T00:59:36.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-09-18T00:59:36.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:36.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:36.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-09-18T00:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T00:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T00:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T00:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T00:59:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T00:59:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T00:59:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T00:59:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T00:59:36.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:36.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:36.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T00:59:36.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-09-18T00:59:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 8 2024-09-18T00:59:44.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T00:59:44.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T00:59:44.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:44.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T00:59:44.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T00:59:44.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:44.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T00:59:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T00:59:44.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T00:59:44.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:44.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:44.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T00:59:44.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T00:59:44.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T00:59:44.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T00:59:44.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:44.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T00:59:44.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T00:59:44.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T00:59:44.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T00:59:44.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T00:59:44.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T00:59:44.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T00:59:44.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:44.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:44.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T00:59:44.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T00:59:44.424 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T00:59:44.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T00:59:44.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T00:59:44.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T00:59:44.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T00:59:44.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T00:59:44.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T00:59:44.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T00:59:44.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ clean_snap_name= 2024-09-18T00:59:44.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 10 2024-09-18T00:59:44.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T00:59:44.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror test 2024-09-18T00:59:44.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T00:59:44.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:44.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T00:59:44.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T00:59:44.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=11 2024-09-18T00:59:44.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T00:59:44.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-18T00:59:44.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 41014272 bytes 2024-09-18T00:59:44.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 117710848 bytes 2024-09-18T00:59:44.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 164846592 bytes 2024-09-18T00:59:45.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 217990144 bytes 2024-09-18T00:59:45.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 302840832 bytes 2024-09-18T00:59:45.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 372458496 bytes 2024-09-18T00:59:45.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 450242560 bytes 2024-09-18T00:59:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 489946112 bytes 2024-09-18T00:59:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 538290176 bytes 2024-09-18T00:59:45.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 563970048 bytes 2024-09-18T00:59:45.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 571544576 bytes 2024-09-18T00:59:45.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 589122560 bytes 2024-09-18T00:59:45.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 628593664 bytes 2024-09-18T00:59:45.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 646776832 bytes 2024-09-18T00:59:45.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 693780480 bytes 2024-09-18T00:59:45.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 736824320 bytes 2024-09-18T00:59:46.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 758082560 bytes 2024-09-18T00:59:46.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 821553152 bytes 2024-09-18T00:59:46.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 872452096 bytes 2024-09-18T00:59:46.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 894570496 bytes 2024-09-18T00:59:46.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 933552128 bytes 2024-09-18T00:59:46.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 980548608 bytes 2024-09-18T00:59:46.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1042398208 bytes 2024-09-18T00:59:46.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1092114432 bytes 2024-09-18T00:59:46.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1132043264 bytes 2024-09-18T00:59:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1178094592 bytes 2024-09-18T00:59:46.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1229041664 bytes 2024-09-18T00:59:47.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1244093440 bytes 2024-09-18T00:59:47.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1249590272 bytes 2024-09-18T00:59:47.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1269608448 bytes 2024-09-18T00:59:47.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1344453632 bytes 2024-09-18T00:59:47.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1385620480 bytes 2024-09-18T00:59:47.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1406828544 bytes 2024-09-18T00:59:47.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1488302080 bytes 2024-09-18T00:59:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1577389056 bytes 2024-09-18T00:59:47.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1591182336 bytes 2024-09-18T00:59:47.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1677717504 bytes 2024-09-18T00:59:47.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1678319616 bytes 2024-09-18T00:59:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1709038592 bytes 2024-09-18T00:59:48.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1729864704 bytes 2024-09-18T00:59:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1798258688 bytes 2024-09-18T00:59:48.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1881096192 bytes 2024-09-18T00:59:48.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1892098048 bytes 2024-09-18T00:59:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1933248512 bytes 2024-09-18T00:59:48.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1955751936 bytes 2024-09-18T00:59:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2000439296 bytes 2024-09-18T00:59:48.783 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2054140928 bytes 2024-09-18T00:59:48.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2072969216 bytes 2024-09-18T00:59:48.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2117545984 bytes 2024-09-18T00:59:49.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2133580800 bytes 2024-09-18T00:59:49.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2138964992 bytes 2024-09-18T00:59:49.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2213430272 bytes 2024-09-18T00:59:49.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2267811840 bytes 2024-09-18T00:59:49.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2317212672 bytes 2024-09-18T00:59:49.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2362329088 bytes 2024-09-18T00:59:49.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2375750656 bytes 2024-09-18T00:59:49.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2430721024 bytes 2024-09-18T00:59:49.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2546589696 bytes 2024-09-18T00:59:49.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2625649664 bytes 2024-09-18T00:59:50.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2717688832 bytes 2024-09-18T00:59:50.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2739521536 bytes 2024-09-18T00:59:50.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2743880704 bytes 2024-09-18T00:59:50.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2794935296 bytes 2024-09-18T00:59:50.406 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2826224640 bytes 2024-09-18T00:59:50.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2871981056 bytes 2024-09-18T00:59:50.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2900293632 bytes 2024-09-18T00:59:50.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2943660032 bytes 2024-09-18T00:59:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3064720384 bytes 2024-09-18T00:59:50.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3102323712 bytes 2024-09-18T00:59:51.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3185885184 bytes 2024-09-18T00:59:51.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3259164672 bytes 2024-09-18T00:59:51.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3300525056 bytes 2024-09-18T00:59:51.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3350325248 bytes 2024-09-18T00:59:51.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3371533312 bytes 2024-09-18T00:59:51.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3409015808 bytes 2024-09-18T00:59:51.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3475138560 bytes 2024-09-18T00:59:51.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3499079680 bytes 2024-09-18T00:59:51.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3519807488 bytes 2024-09-18T00:59:51.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3520431104 bytes 2024-09-18T00:59:51.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3521037312 bytes 2024-09-18T00:59:51.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3557793792 bytes 2024-09-18T00:59:52.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3656566784 bytes 2024-09-18T00:59:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3718377472 bytes 2024-09-18T00:59:52.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3760484352 bytes 2024-09-18T00:59:52.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3818340352 bytes 2024-09-18T00:59:52.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3819875328 bytes 2024-09-18T00:59:52.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3864154112 bytes 2024-09-18T00:59:52.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3922088960 bytes 2024-09-18T00:59:52.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3944317952 bytes 2024-09-18T00:59:52.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3952196608 bytes 2024-09-18T00:59:52.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3967990784 bytes 2024-09-18T00:59:52.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4058847232 bytes 2024-09-18T00:59:53.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4091329536 bytes 2024-09-18T00:59:53.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4138485760 bytes 2024-09-18T00:59:53.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4202851328 bytes 2024-09-18T00:59:53.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4218411008 bytes 2024-09-18T00:59:53.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4255104000 bytes 2024-09-18T00:59:53.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4337468416 bytes 2024-09-18T00:59:53.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4353940480 bytes 2024-09-18T00:59:53.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4399136768 bytes 2024-09-18T00:59:54.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4458259456 bytes 2024-09-18T00:59:54.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4469214208 bytes 2024-09-18T00:59:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4513482752 bytes 2024-09-18T00:59:54.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4537380864 bytes 2024-09-18T00:59:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4579708928 bytes 2024-09-18T00:59:54.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4628974592 bytes 2024-09-18T00:59:54.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4693180416 bytes 2024-09-18T00:59:54.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4739866624 bytes 2024-09-18T00:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4769912832 bytes 2024-09-18T00:59:54.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 4831029248 bytes 2024-09-18T00:59:54.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11100 IOs, 4831668224 bytes 2024-09-18T00:59:55.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11200 IOs, 4859373568 bytes 2024-09-18T00:59:55.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11300 IOs, 4921647104 bytes 2024-09-18T00:59:55.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11400 IOs, 4971396096 bytes 2024-09-18T00:59:55.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11500 IOs, 5027860480 bytes 2024-09-18T00:59:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T00:59:55.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T00:59:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T00:59:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T00:59:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-18T00:59:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T00:59:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T00:59:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T00:59:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T00:59:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:55.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T00:59:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T00:59:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T00:59:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T00:59:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T00:59:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T00:59:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T00:59:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T00:59:56.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T00:59:56.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T00:59:56.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T00:59:56.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: down+unknown 2024-09-18T00:59:56.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: status not found 2024-09-18T00:59:56.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18T00:59:56.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T00:59:56.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T00:59:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: down+unknown 2024-09-18T00:59:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: status not found 2024-09-18T00:59:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: ' 2024-09-18T00:59:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:59:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 1 2024-09-18T00:59:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 2 2024-09-18T00:59:58.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T00:59:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T00:59:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T00:59:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T00:59:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T00:59:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T00:59:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T00:59:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T00:59:58.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T00:59:58.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T00:59:58.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T00:59:58.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T00:59:58.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: down+unknown 2024-09-18T00:59:58.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: status not found 2024-09-18T00:59:58.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18T00:59:58.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T00:59:58.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T00:59:58.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: down+unknown 2024-09-18T00:59:58.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: status not found 2024-09-18T00:59:58.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: ' 2024-09-18T00:59:58.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T00:59:58.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 1 2024-09-18T00:59:58.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T00:59:58.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 4 2024-09-18T01:00:02.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T01:00:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:00:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:00:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:00:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:00:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:00:02.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T01:00:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T01:00:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:00:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: down+unknown 2024-09-18T01:00:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: status not found 2024-09-18T01:00:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18T01:00:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T01:00:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T01:00:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: down+unknown 2024-09-18T01:00:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: status not found 2024-09-18T01:00:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: ' 2024-09-18T01:00:02.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:00:02.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 1 2024-09-18T01:00:02.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:00:02.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 8 2024-09-18T01:00:10.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T01:00:10.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:10.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:10.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:10.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:00:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:00:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:00:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:00:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:00:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T01:00:10.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T01:00:10.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:00:10.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+replaying 2024-09-18T01:00:10.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621158,"remote_snapshot_timestamp":1726621158,"replay_state":"idle"} 2024-09-18T01:00:10.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: service: mirror.1 on smithi150 2024-09-18T01:00:10.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:00:03 2024-09-18T01:00:10.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T01:00:10.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T01:00:10.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+stopped 2024-09-18T01:00:10.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: local image is primary 2024-09-18T01:00:10.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:00:03' 2024-09-18T01:00:10.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:00:10.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: state: up+replaying 2024-09-18T01:00:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ description: .*' 2024-09-18T01:00:10.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621158,"remote_snapshot_timestamp":1726621158,"replay_state":"idle"} 2024-09-18T01:00:10.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:00:10.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:00:10.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^up+' 2024-09-18T01:00:10.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:00:10.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:00:10.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: service: mirror.1 on smithi150 2024-09-18T01:00:10.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update 2024-09-18T01:00:10.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:00:10.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:00:03' 2024-09-18T01:00:10.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-18 01:00:03' 2024-09-18T01:00:10.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:10.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:10.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:10.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:00:10.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local 'prev_last_update=2024-09-18 01:00:03' 2024-09-18T01:00:10.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:00:10.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:00:10.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:00:10.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='OKOKOK142094291mirror.0smithi15019.1.1-330-gc2ddf972falseOK42134284mirror.1smithi15019.1.1-330-gc2ddf972trueOK42174290mirror.2smithi15019.1.1-330-gc2ddf972falseOK42214285mirror.3smithi15019.1.1-330-gc2ddf972falseOKtestc32f6c34-fc35-4674-a05b-f4b6c1db7403up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621158,"remote_snapshot_timestamp":1726621158,"replay_state":"idle"}42134284mirror.1smithi1502024-09-18 01:00:03cluster2b28075f7-24f5-441c-873b-85d148cb477bup+stoppedlocal image is primary2024-09-18 01:00:03' 2024-09-18T01:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update state 2024-09-18T01:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-18T01:00:10.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:00:03' 2024-09-18T01:00:10.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-18T01:00:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=up+replaying 2024-09-18T01:00:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:00:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep up+replaying 2024-09-18T01:00:10.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:00:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:00:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:00:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap1 2024-09-18T01:00:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap1 2024-09-18T01:00:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:00:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap1 2024-09-18T01:00:10.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:00:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T01:00:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T01:00:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:27.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:27.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:00:27.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:00:27.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:00:27.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:00:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T01:00:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:28.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:00:28.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:00:28.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:00:28.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:00:28.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:00:28.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:00:28.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T01:00:28.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:00:28.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:00:28.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:00:28.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:00:28.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:00:28.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:00:28.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:00:28.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:00:28.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:00:28.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:00:28.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:00:28.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:00:28.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:00:28.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:00:28.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:00:28.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:00:28.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:00:28.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:00:28.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:00:28.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:00:28.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:00:28.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:00:28.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T01:00:28.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T01:00:28.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:00:28.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:00:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:00:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T01:00:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:00:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:00:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:00:28.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:00:28.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:00:28.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:00:28.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:00:28.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:00:28.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:00:28.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T01:00:28.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:00:28.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:00:28.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:00:28.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:00:28.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:00:28.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:00:28.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:00:28.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:00:28.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-18T01:00:28.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:00:28.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-test.status 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-test.status 2024-09-18T01:00:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:00:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:00:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-test.status 2024-09-18T01:00:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-test.status 2024-09-18T01:00:28.406 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:00:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:00:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-18T01:00:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:00:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:28.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-18T01:00:29.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 5 2024-09-18T01:00:29.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:00:29.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:00:29.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:29.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:29.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:00:29.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-18T01:00:29.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:00:29.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:00:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=5 2024-09-18T01:00:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:00:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:00:29.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:00:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:00:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:00:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:00:29.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:00:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:00:29.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=3 2024-09-18T01:00:29.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 5 = 3 2024-09-18T01:00:29.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:00:29.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:00:29.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:00:29.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:29.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:29.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:29.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:00:29.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:00:29.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:00:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:00:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:00:29.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:00:29.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:00:29.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:00:29.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:00:29.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 5 = '' 2024-09-18T01:00:29.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:00:29.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:00:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:00:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:00:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:00:30.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:00:30.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:00:30.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:00:30.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:00:30.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:00:30.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:00:30.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:00:30.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 5 = '' 2024-09-18T01:00:30.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:00:30.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:00:32.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:00:32.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:32.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:32.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:32.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:00:32.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:00:32.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:00:32.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:00:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=5 2024-09-18T01:00:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 5 = 5 2024-09-18T01:00:32.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:00:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror test snap1 2024-09-18T01:00:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap1 2024-09-18T01:00:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:00:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:00:32.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:00:33.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/test@snap1 2024-09-18T01:00:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T01:00:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:00:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:00:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:00:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc231b3fe3 2024-09-18T01:00:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc231b3fe3 2024-09-18T01:00:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:00:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:00:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:00:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:00:33.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:00:33.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:00:33.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:00:33.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:00:33.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:00:33.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:00:33.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:00:33.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:00:33.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:00:33.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:00:33.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap1 2024-09-18T01:00:33.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:33.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:33.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap1 2024-09-18T01:00:33.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:00:33.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:00:33.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:00:33.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:00:33.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:00:34.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:00:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:00:34.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:00:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:00:34.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:00:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:00:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ clean_snap_name=snap1-clean 2024-09-18T01:00:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap1-clean 2024-09-18T01:00:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:00:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap1-clean 2024-09-18T01:00:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20 2024-09-18T01:00:35.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:00:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror test 2024-09-18T01:00:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:00:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:35.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:00:35.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=7 2024-09-18T01:00:35.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:00:35.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-18T01:00:35.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 11019264 bytes 2024-09-18T01:00:35.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 40797184 bytes 2024-09-18T01:00:35.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 110273536 bytes 2024-09-18T01:00:35.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 148856832 bytes 2024-09-18T01:00:35.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 179640320 bytes 2024-09-18T01:00:35.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 180253696 bytes 2024-09-18T01:00:35.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 220199936 bytes 2024-09-18T01:00:36.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 294749184 bytes 2024-09-18T01:00:36.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 319381504 bytes 2024-09-18T01:00:36.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 364803072 bytes 2024-09-18T01:00:36.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 368633856 bytes 2024-09-18T01:00:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 383721472 bytes 2024-09-18T01:00:36.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 399800320 bytes 2024-09-18T01:00:36.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 436056064 bytes 2024-09-18T01:00:36.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 458552320 bytes 2024-09-18T01:00:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 491887616 bytes 2024-09-18T01:00:36.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 522995712 bytes 2024-09-18T01:00:36.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 558963712 bytes 2024-09-18T01:00:36.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 581057536 bytes 2024-09-18T01:00:37.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 638750720 bytes 2024-09-18T01:00:37.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 710751232 bytes 2024-09-18T01:00:37.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 754519040 bytes 2024-09-18T01:00:37.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 801547264 bytes 2024-09-18T01:00:37.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 823267328 bytes 2024-09-18T01:00:37.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 844725248 bytes 2024-09-18T01:00:37.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 873149440 bytes 2024-09-18T01:00:37.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 910821376 bytes 2024-09-18T01:00:37.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 925175808 bytes 2024-09-18T01:00:37.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 993669120 bytes 2024-09-18T01:00:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1054592000 bytes 2024-09-18T01:00:38.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1098381312 bytes 2024-09-18T01:00:38.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1120995328 bytes 2024-09-18T01:00:38.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1121591296 bytes 2024-09-18T01:00:38.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1155471360 bytes 2024-09-18T01:00:38.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1206836224 bytes 2024-09-18T01:00:38.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1237687296 bytes 2024-09-18T01:00:38.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1242868736 bytes 2024-09-18T01:00:38.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1265601536 bytes 2024-09-18T01:00:38.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1285592064 bytes 2024-09-18T01:00:38.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1297656832 bytes 2024-09-18T01:00:38.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1334547456 bytes 2024-09-18T01:00:38.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1374595072 bytes 2024-09-18T01:00:38.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1433265152 bytes 2024-09-18T01:00:39.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1489913856 bytes 2024-09-18T01:00:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1536093184 bytes 2024-09-18T01:00:39.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1565329408 bytes 2024-09-18T01:00:39.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1646714880 bytes 2024-09-18T01:00:39.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1761824768 bytes 2024-09-18T01:00:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1821586432 bytes 2024-09-18T01:00:39.783 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1935323136 bytes 2024-09-18T01:00:39.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2008393728 bytes 2024-09-18T01:00:40.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2048149504 bytes 2024-09-18T01:00:40.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2068755456 bytes 2024-09-18T01:00:40.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2127705088 bytes 2024-09-18T01:00:40.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2232892416 bytes 2024-09-18T01:00:40.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2270983168 bytes 2024-09-18T01:00:40.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2298036224 bytes 2024-09-18T01:00:40.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2416076800 bytes 2024-09-18T01:00:40.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2455591936 bytes 2024-09-18T01:00:40.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2496920576 bytes 2024-09-18T01:00:40.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2497534976 bytes 2024-09-18T01:00:40.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2522601472 bytes 2024-09-18T01:00:41.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2563238912 bytes 2024-09-18T01:00:41.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2604055552 bytes 2024-09-18T01:00:41.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2659831808 bytes 2024-09-18T01:00:41.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2716397568 bytes 2024-09-18T01:00:41.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2735328256 bytes 2024-09-18T01:00:41.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2789233664 bytes 2024-09-18T01:00:41.658 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2856199168 bytes 2024-09-18T01:00:41.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2899372032 bytes 2024-09-18T01:00:41.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2939803648 bytes 2024-09-18T01:00:41.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2950855680 bytes 2024-09-18T01:00:42.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3042287616 bytes 2024-09-18T01:00:42.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3055627264 bytes 2024-09-18T01:00:42.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3097725952 bytes 2024-09-18T01:00:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:00:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:00:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:00:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:00:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-18T01:00:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:42.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:00:42.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:00:42.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:00:42.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:00:42.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:00:43.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T01:00:43.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:43.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:43.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:43.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:00:43.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:00:43.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:00:43.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:00:43.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:00:43.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T01:00:43.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T01:00:43.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:00:43.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+replaying 2024-09-18T01:00:43.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: replaying, {"bytes_per_second":12443101.87,"bytes_per_snapshot":267564544.0,"last_snapshot_bytes":535129088,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621229,"remote_snapshot_timestamp":1726621229,"replay_state":"idle"} 2024-09-18T01:00:43.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: service: mirror.1 on smithi150 2024-09-18T01:00:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:00:33 2024-09-18T01:00:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T01:00:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T01:00:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+stopped 2024-09-18T01:00:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: local image is primary 2024-09-18T01:00:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:00:33' 2024-09-18T01:00:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:00:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: state: up+replaying 2024-09-18T01:00:43.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ description: .*' 2024-09-18T01:00:43.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: description: replaying, {"bytes_per_second":12443101.87,"bytes_per_snapshot":267564544.0,"last_snapshot_bytes":535129088,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621229,"remote_snapshot_timestamp":1726621229,"replay_state":"idle"} 2024-09-18T01:00:43.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:00:43.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:00:43.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^up+' 2024-09-18T01:00:43.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:00:43.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:00:43.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: service: mirror.1 on smithi150 2024-09-18T01:00:43.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update 2024-09-18T01:00:43.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:00:43.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:00:33' 2024-09-18T01:00:43.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-18 01:00:33' 2024-09-18T01:00:43.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:00:43.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:43.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:43.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:00:43.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local 'prev_last_update=2024-09-18 01:00:33' 2024-09-18T01:00:43.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:00:43.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:00:43.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:00:43.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='OKOKOK142094291mirror.0smithi15019.1.1-330-gc2ddf972falseOK42134284mirror.1smithi15019.1.1-330-gc2ddf972trueOK42174290mirror.2smithi15019.1.1-330-gc2ddf972falseOK42214285mirror.3smithi15019.1.1-330-gc2ddf972falseOKtestc32f6c34-fc35-4674-a05b-f4b6c1db7403up+replayingreplaying, {"bytes_per_second":12443101.87,"bytes_per_snapshot":267564544.0,"last_snapshot_bytes":535129088,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621229,"remote_snapshot_timestamp":1726621229,"replay_state":"idle"}42134284mirror.1smithi1502024-09-18 01:00:33cluster2b28075f7-24f5-441c-873b-85d148cb477bup+stoppedlocal image is primary2024-09-18 01:00:33' 2024-09-18T01:00:43.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update state 2024-09-18T01:00:43.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-18T01:00:43.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:00:33' 2024-09-18T01:00:43.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-18T01:00:43.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=up+replaying 2024-09-18T01:00:43.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:00:43.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep up+replaying 2024-09-18T01:00:43.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:00:43.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:00:43.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:00:43.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap2 2024-09-18T01:00:43.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap2 2024-09-18T01:00:43.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:00:43.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:00:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:00:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap2 2024-09-18T01:00:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:00:56.254 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-18T01:00:56.287 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-18T01:00:56.287 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-09-18T01:00:56.288 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:00:56.286+0000 7f532d4f9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 36988) UID: 1000 2024-09-18T01:01:01.288 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-18T01:01:01.288 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:01:01.288 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-18T01:01:01.288 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-18T01:01:01.289 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-18T01:01:01.292 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-18T01:01:01.292 INFO:tasks.rbd_mirror_thrash:waiting for 107 secs before thrashing 2024-09-18T01:01:12.253 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-18T01:01:12.253 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-18T01:01:12.254 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-18T01:01:12.254 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-18T01:01:12.254 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:01:12.255 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:01:12.254+0000 7fab65768640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 37125) UID: 1000 2024-09-18T01:01:13.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T01:01:13.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:01:13.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:13.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:13.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T01:01:13.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:01:13.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:13.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:13.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:01:13.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:01:13.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:01:13.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:01:14.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T01:01:14.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:01:14.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:14.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:14.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:01:14.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:01:14.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:01:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T01:01:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:01:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:01:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:01:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:01:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:01:14.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:01:14.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:01:14.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:01:14.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:01:14.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:01:14.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:01:14.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T01:01:14.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:01:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:01:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:01:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:01:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:01:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:01:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:01:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:01:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:01:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:01:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:01:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:01:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:01:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:01:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:01:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:01:14.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:01:14.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:01:14.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:01:14.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:01:14.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:01:14.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:01:14.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:01:14.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:01:14.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:01:14.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:01:14.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:01:14.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:01:14.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:01:14.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:01:14.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:01:14.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:01:14.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:01:14.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T01:01:14.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T01:01:14.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:01:14.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:01:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:01:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T01:01:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:01:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:01:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:01:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:01:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:01:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:01:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:01:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:01:14.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:01:14.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:01:14.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:01:14.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:01:14.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:01:14.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:01:14.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:01:14.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:01:14.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:01:14.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:01:14.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:01:14.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:01:14.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T01:01:14.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:01:14.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:01:14.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:01:14.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:01:14.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:01:14.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:01:14.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:01:14.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:01:14.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-18T01:01:14.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:01:14.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:01:14.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:01:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:01:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-18T01:01:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:01:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:01:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-test.status 2024-09-18T01:01:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-test.status 2024-09-18T01:01:14.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:01:14.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:01:14.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-test.status 2024-09-18T01:01:14.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-test.status 2024-09-18T01:01:14.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:01:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:01:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-18T01:01:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:01:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:14.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-18T01:01:16.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 8 2024-09-18T01:01:16.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:01:16.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:01:16.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:16.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:16.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:01:16.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-18T01:01:16.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:01:16.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:01:16.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=8 2024-09-18T01:01:16.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:01:16.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:01:16.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:01:16.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:01:16.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:01:16.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:16.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:16.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:01:16.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:01:16.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:01:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:01:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:01:16.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:01:16.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:01:16.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:01:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:01:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 8 = '' 2024-09-18T01:01:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:01:16.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:01:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:01:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:01:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:01:16.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:01:16.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:01:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:01:17.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:01:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:01:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:01:17.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:01:17.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:01:17.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 8 = '' 2024-09-18T01:01:17.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:01:17.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:01:17.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:01:17.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:01:17.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:17.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:17.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:01:17.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:01:17.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:01:17.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:01:17.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:01:17.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:01:17.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:01:17.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:01:17.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:01:17.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 8 = '' 2024-09-18T01:01:17.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:01:17.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:01:18.355 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-18T01:01:18.356 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-18T01:01:18.356 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-18T01:01:18.359 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-18T01:01:18.359 INFO:tasks.rbd_mirror_thrash:waiting for 75 secs before thrashing 2024-09-18T01:01:19.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:01:19.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:01:19.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:19.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:19.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:01:19.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:01:19.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:01:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:01:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=8 2024-09-18T01:01:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 8 = 8 2024-09-18T01:01:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:01:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror test snap2 2024-09-18T01:01:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:01:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap2 2024-09-18T01:01:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:01:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:01:19.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:01:20.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/test@snap2 2024-09-18T01:01:20.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T01:01:20.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 4 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc231b3fe3 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc231b3fe3 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:01:20.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:01:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:01:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:01:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:01:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:01:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snap1-clean ']' 2024-09-18T01:01:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap1-clean 2024-09-18T01:01:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap1-clean 2024-09-18T01:01:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:01:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:01:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:01:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:01:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:01:21.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:01:21.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:01:21.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:01:21.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:01:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:01:22.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:01:22.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap2 2024-09-18T01:01:22.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:22.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:22.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap2 2024-09-18T01:01:22.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:01:22.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:01:22.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:01:22.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:01:22.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:01:22.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:01:22.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:01:23.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:01:23.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:01:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:01:23.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:01:23.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ clean_snap_name=snap2-clean 2024-09-18T01:01:23.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap2-clean 2024-09-18T01:01:23.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:01:23.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:23.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:23.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap2-clean 2024-09-18T01:01:23.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20 2024-09-18T01:01:24.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:01:24.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror test 2024-09-18T01:01:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:01:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:24.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:01:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=6 2024-09-18T01:01:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:01:24.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-18T01:01:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 21595136 bytes 2024-09-18T01:01:24.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 91211776 bytes 2024-09-18T01:01:24.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 139630592 bytes 2024-09-18T01:01:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 195562496 bytes 2024-09-18T01:01:24.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 243534848 bytes 2024-09-18T01:01:25.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 324834304 bytes 2024-09-18T01:01:25.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 351273984 bytes 2024-09-18T01:01:25.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 374117376 bytes 2024-09-18T01:01:25.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 445579264 bytes 2024-09-18T01:01:25.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 478798848 bytes 2024-09-18T01:01:25.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 591448064 bytes 2024-09-18T01:01:25.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 600301568 bytes 2024-09-18T01:01:25.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 600913920 bytes 2024-09-18T01:01:25.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 623378432 bytes 2024-09-18T01:01:25.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 691437568 bytes 2024-09-18T01:01:25.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 722042880 bytes 2024-09-18T01:01:26.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 780792832 bytes 2024-09-18T01:01:26.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 883533824 bytes 2024-09-18T01:01:26.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 911788032 bytes 2024-09-18T01:01:26.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 956948480 bytes 2024-09-18T01:01:26.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1002151936 bytes 2024-09-18T01:01:26.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1057104896 bytes 2024-09-18T01:01:26.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1099958272 bytes 2024-09-18T01:01:26.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1118949376 bytes 2024-09-18T01:01:26.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1119562752 bytes 2024-09-18T01:01:26.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1200359424 bytes 2024-09-18T01:01:27.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1266613248 bytes 2024-09-18T01:01:27.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1318484992 bytes 2024-09-18T01:01:27.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1345326080 bytes 2024-09-18T01:01:27.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1368681472 bytes 2024-09-18T01:01:27.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1409069056 bytes 2024-09-18T01:01:27.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1461092352 bytes 2024-09-18T01:01:27.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1497665536 bytes 2024-09-18T01:01:27.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1522270208 bytes 2024-09-18T01:01:27.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1527391232 bytes 2024-09-18T01:01:27.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1622774784 bytes 2024-09-18T01:01:28.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1720598528 bytes 2024-09-18T01:01:28.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1736731648 bytes 2024-09-18T01:01:28.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1834182656 bytes 2024-09-18T01:01:28.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1852597248 bytes 2024-09-18T01:01:28.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1885493248 bytes 2024-09-18T01:01:28.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1906816000 bytes 2024-09-18T01:01:28.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1930607616 bytes 2024-09-18T01:01:28.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2001764352 bytes 2024-09-18T01:01:28.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2058826752 bytes 2024-09-18T01:01:28.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2106933248 bytes 2024-09-18T01:01:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2158055424 bytes 2024-09-18T01:01:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2173974528 bytes 2024-09-18T01:01:29.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2237371392 bytes 2024-09-18T01:01:29.338 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2287422464 bytes 2024-09-18T01:01:29.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2359268352 bytes 2024-09-18T01:01:29.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2381454336 bytes 2024-09-18T01:01:29.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2429172736 bytes 2024-09-18T01:01:29.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2476804096 bytes 2024-09-18T01:01:29.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2532177920 bytes 2024-09-18T01:01:29.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2549627904 bytes 2024-09-18T01:01:29.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2552302592 bytes 2024-09-18T01:01:30.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2552901632 bytes 2024-09-18T01:01:30.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2637444096 bytes 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:01:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:01:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:01:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:01:30.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:01:31.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T01:01:31.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:01:31.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:31.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:31.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:01:31.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:01:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:01:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:01:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:01:31.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T01:01:31.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T01:01:31.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:01:31.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+replaying 2024-09-18T01:01:31.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: replaying, {"bytes_per_second":17837636.27,"bytes_per_snapshot":267564544.0,"last_snapshot_bytes":535129088,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621229,"remote_snapshot_timestamp":1726621229,"replay_state":"idle"} 2024-09-18T01:01:31.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: service: mirror.1 on smithi150 2024-09-18T01:01:31.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:01:03 2024-09-18T01:01:31.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T01:01:31.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T01:01:31.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: down+unknown 2024-09-18T01:01:31.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: status not found 2024-09-18T01:01:31.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: ' 2024-09-18T01:01:31.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:01:31.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: state: up+replaying 2024-09-18T01:01:31.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ description: .*' 2024-09-18T01:01:31.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: description: replaying, {"bytes_per_second":17837636.27,"bytes_per_snapshot":267564544.0,"last_snapshot_bytes":535129088,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621229,"remote_snapshot_timestamp":1726621229,"replay_state":"idle"} 2024-09-18T01:01:31.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:01:31.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:01:31.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^up+' 2024-09-18T01:01:31.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:01:31.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:01:31.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: service: mirror.1 on smithi150 2024-09-18T01:01:31.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update 2024-09-18T01:01:31.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:01:31.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:01:03' 2024-09-18T01:01:31.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-18 01:01:03' 2024-09-18T01:01:31.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:01:31.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:31.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:31.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:01:31.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local 'prev_last_update=2024-09-18 01:01:03' 2024-09-18T01:01:31.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:01:31.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:01:31.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:01:31.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='WARNINGOKWARNING142134284mirror.1smithi15019.1.1-330-gc2ddf972trueOK42174290mirror.2smithi15019.1.1-330-gc2ddf972falseOK42214285mirror.3smithi15019.1.1-330-gc2ddf972falseOK43444347mirror.0smithi15019.1.1-330-gc2ddf972falseOKtestc32f6c34-fc35-4674-a05b-f4b6c1db7403up+replayingreplaying, {"bytes_per_second":17837636.27,"bytes_per_snapshot":267564544.0,"last_snapshot_bytes":535129088,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621229,"remote_snapshot_timestamp":1726621229,"replay_state":"idle"}42134284mirror.1smithi1502024-09-18 01:01:03cluster2b28075f7-24f5-441c-873b-85d148cb477bdown+unknownstatus not found' 2024-09-18T01:01:31.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update state 2024-09-18T01:01:31.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-18T01:01:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:01:03' 2024-09-18T01:01:31.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-18T01:01:31.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=up+replaying 2024-09-18T01:01:31.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:01:31.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep up+replaying 2024-09-18T01:01:31.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:01:31.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:01:31.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:01:31.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap3 2024-09-18T01:01:31.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap3 2024-09-18T01:01:31.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:01:31.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:01:31.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:01:31.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap3 2024-09-18T01:01:31.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:02:01.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T01:02:01.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:01.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:01.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:01.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T01:02:01.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:01.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:01.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:01.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:02:01.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:02:01.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:02:01.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:02:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T01:02:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:02:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:02:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:02:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T01:02:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:02:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:02:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:02:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:02:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:02:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:02:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:02:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:02:02.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:02:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:02:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:02:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T01:02:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:02:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:02:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:02:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:02:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:02:02.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:02:02.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:02:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:02:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:02:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:02:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:02:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:02:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:02:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:02:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:02:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:02:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:02:02.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:02:02.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:02:02.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:02:02.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T01:02:02.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T01:02:02.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:02:02.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:02:02.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:02:02.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T01:02:02.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:02:02.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:02:02.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:02:02.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:02:02.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:02:02.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:02:02.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:02:02.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:02:02.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T01:02:03.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:02:03.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:02:03.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:02:03.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:02:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:02:03.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:02:03.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:02:03.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-test.status 2024-09-18T01:02:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-test.status 2024-09-18T01:02:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:02:03.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:02:03.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-test.status 2024-09-18T01:02:03.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-test.status 2024-09-18T01:02:03.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:02:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:02:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-18T01:02:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:02:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:03.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-18T01:02:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 11 2024-09-18T01:02:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:02:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:02:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:02:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-18T01:02:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:02:04.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:02:04.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=11 2024-09-18T01:02:04.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:02:04.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:02:04.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:02:05.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:02:05.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:05.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:05.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:05.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:02:05.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:02:05.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:02:05.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:02:05.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:02:05.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:02:05.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:02:05.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:02:05.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:02:05.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 11 = '' 2024-09-18T01:02:05.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:02:05.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:02:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:02:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:02:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:02:05.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:02:05.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:02:05.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:02:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:02:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:02:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:02:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:02:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 11 = '' 2024-09-18T01:02:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:02:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:02:06.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:02:06.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:06.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:06.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:06.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:02:06.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:02:06.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:02:06.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:02:06.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:02:06.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:02:06.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:02:06.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:02:06.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:02:06.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 11 = '' 2024-09-18T01:02:06.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:02:06.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:02:08.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:02:08.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:08.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:08.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:08.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:02:08.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:02:08.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:02:08.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:02:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=11 2024-09-18T01:02:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 11 = 11 2024-09-18T01:02:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:02:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror test snap3 2024-09-18T01:02:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap3 2024-09-18T01:02:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:02:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:02:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:02:09.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/test@snap3 2024-09-18T01:02:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T01:02:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:02:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:02:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 6 2024-09-18T01:02:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc231b3fe3 2024-09-18T01:02:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc231b3fe3 2024-09-18T01:02:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:02:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:02:09.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:02:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:02:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:02:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:01:21 2024 2024-09-18T01:02:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:02:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:02:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:02:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:02:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:02:09.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:02:09.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:02:09.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snap2-clean ']' 2024-09-18T01:02:09.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap2-clean 2024-09-18T01:02:09.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:09.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:09.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap2-clean 2024-09-18T01:02:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:02:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:02:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:02:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:02:09.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:02:09.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:02:09.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:02:10.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:02:10.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:02:10.658 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:02:10.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:02:10.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap3 2024-09-18T01:02:10.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:10.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap3 2024-09-18T01:02:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:02:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:02:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:02:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:02:10.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:02:11.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:02:11.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:02:12.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:02:12.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:02:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:02:12.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:02:12.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ clean_snap_name=snap3-clean 2024-09-18T01:02:12.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap3-clean 2024-09-18T01:02:12.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:02:12.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:12.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:12.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap3-clean 2024-09-18T01:02:12.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20 2024-09-18T01:02:12.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:02:12.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror test 2024-09-18T01:02:12.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:02:12.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:12.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:12.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:02:12.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=13 2024-09-18T01:02:12.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:02:12.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-18T01:02:13.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 68873216 bytes 2024-09-18T01:02:13.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 126657536 bytes 2024-09-18T01:02:13.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 172138496 bytes 2024-09-18T01:02:13.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 212139008 bytes 2024-09-18T01:02:13.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 258908160 bytes 2024-09-18T01:02:13.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 282245120 bytes 2024-09-18T01:02:13.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 319689728 bytes 2024-09-18T01:02:13.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 430862336 bytes 2024-09-18T01:02:13.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 520425472 bytes 2024-09-18T01:02:14.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 532117504 bytes 2024-09-18T01:02:14.151 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 623334400 bytes 2024-09-18T01:02:14.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 661450752 bytes 2024-09-18T01:02:14.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 699275264 bytes 2024-09-18T01:02:14.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 699891712 bytes 2024-09-18T01:02:14.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 726921216 bytes 2024-09-18T01:02:14.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 748414976 bytes 2024-09-18T01:02:14.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 777354240 bytes 2024-09-18T01:02:14.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 807116800 bytes 2024-09-18T01:02:14.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 848630784 bytes 2024-09-18T01:02:14.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 875884544 bytes 2024-09-18T01:02:15.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 916412416 bytes 2024-09-18T01:02:15.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 977244160 bytes 2024-09-18T01:02:15.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 992828416 bytes 2024-09-18T01:02:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1076517888 bytes 2024-09-18T01:02:15.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1112121344 bytes 2024-09-18T01:02:15.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1118259200 bytes 2024-09-18T01:02:15.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1162421248 bytes 2024-09-18T01:02:15.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1163030528 bytes 2024-09-18T01:02:15.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1176309760 bytes 2024-09-18T01:02:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1270841344 bytes 2024-09-18T01:02:15.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1296147456 bytes 2024-09-18T01:02:16.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1358526464 bytes 2024-09-18T01:02:16.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1399205888 bytes 2024-09-18T01:02:16.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1405665280 bytes 2024-09-18T01:02:16.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1476283392 bytes 2024-09-18T01:02:16.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1533528064 bytes 2024-09-18T01:02:16.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1560620032 bytes 2024-09-18T01:02:16.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1635899392 bytes 2024-09-18T01:02:16.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1703314432 bytes 2024-09-18T01:02:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1736527872 bytes 2024-09-18T01:02:16.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1769227264 bytes 2024-09-18T01:02:17.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1803273216 bytes 2024-09-18T01:02:17.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1860572160 bytes 2024-09-18T01:02:17.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1942895616 bytes 2024-09-18T01:02:17.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1977190400 bytes 2024-09-18T01:02:17.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2038907904 bytes 2024-09-18T01:02:17.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2080245760 bytes 2024-09-18T01:02:17.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2190689280 bytes 2024-09-18T01:02:17.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2238498816 bytes 2024-09-18T01:02:17.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2269157376 bytes 2024-09-18T01:02:18.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2291483648 bytes 2024-09-18T01:02:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2310890496 bytes 2024-09-18T01:02:18.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2398215168 bytes 2024-09-18T01:02:18.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2416715776 bytes 2024-09-18T01:02:18.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2520069120 bytes 2024-09-18T01:02:18.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2574446592 bytes 2024-09-18T01:02:18.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2592834560 bytes 2024-09-18T01:02:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2626301952 bytes 2024-09-18T01:02:18.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2633019392 bytes 2024-09-18T01:02:18.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2649489408 bytes 2024-09-18T01:02:19.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2712110080 bytes 2024-09-18T01:02:19.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2756277248 bytes 2024-09-18T01:02:19.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2781067264 bytes 2024-09-18T01:02:19.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2842458112 bytes 2024-09-18T01:02:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2863516672 bytes 2024-09-18T01:02:19.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2873481216 bytes 2024-09-18T01:02:19.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2918642688 bytes 2024-09-18T01:02:19.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2958639104 bytes 2024-09-18T01:02:19.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2973785088 bytes 2024-09-18T01:02:19.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3014680576 bytes 2024-09-18T01:02:19.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3076869120 bytes 2024-09-18T01:02:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3123828736 bytes 2024-09-18T01:02:20.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3152493568 bytes 2024-09-18T01:02:20.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3213464576 bytes 2024-09-18T01:02:20.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3275680768 bytes 2024-09-18T01:02:20.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3329981440 bytes 2024-09-18T01:02:20.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3371111424 bytes 2024-09-18T01:02:20.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3477384192 bytes 2024-09-18T01:02:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3564539904 bytes 2024-09-18T01:02:20.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3593587712 bytes 2024-09-18T01:02:21.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3656403968 bytes 2024-09-18T01:02:21.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3684941824 bytes 2024-09-18T01:02:21.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3723674624 bytes 2024-09-18T01:02:21.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3783900160 bytes 2024-09-18T01:02:21.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3784530944 bytes 2024-09-18T01:02:21.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3874050048 bytes 2024-09-18T01:02:21.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3912817664 bytes 2024-09-18T01:02:21.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3934269440 bytes 2024-09-18T01:02:21.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3975874560 bytes 2024-09-18T01:02:21.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 4026748928 bytes 2024-09-18T01:02:22.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 4063139840 bytes 2024-09-18T01:02:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4116532224 bytes 2024-09-18T01:02:22.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4156107776 bytes 2024-09-18T01:02:22.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4172771328 bytes 2024-09-18T01:02:22.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4269004800 bytes 2024-09-18T01:02:22.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4288598016 bytes 2024-09-18T01:02:22.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4334906368 bytes 2024-09-18T01:02:22.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4370105344 bytes 2024-09-18T01:02:22.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4430566400 bytes 2024-09-18T01:02:22.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4504833024 bytes 2024-09-18T01:02:23.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4508859392 bytes 2024-09-18T01:02:23.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4509452288 bytes 2024-09-18T01:02:23.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4554688512 bytes 2024-09-18T01:02:23.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4570888192 bytes 2024-09-18T01:02:23.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4644880384 bytes 2024-09-18T01:02:23.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4664368128 bytes 2024-09-18T01:02:23.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4732413952 bytes 2024-09-18T01:02:23.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4784819200 bytes 2024-09-18T01:02:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4832920576 bytes 2024-09-18T01:02:23.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 4869054464 bytes 2024-09-18T01:02:24.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11100 IOs, 4940684288 bytes 2024-09-18T01:02:24.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11200 IOs, 4971072512 bytes 2024-09-18T01:02:24.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11300 IOs, 5027565568 bytes 2024-09-18T01:02:24.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11400 IOs, 5061469184 bytes 2024-09-18T01:02:24.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11500 IOs, 5140856832 bytes 2024-09-18T01:02:24.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11600 IOs, 5212541952 bytes 2024-09-18T01:02:24.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11700 IOs, 5243347968 bytes 2024-09-18T01:02:24.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11800 IOs, 5273892864 bytes 2024-09-18T01:02:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11900 IOs, 5308910592 bytes 2024-09-18T01:02:24.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12000 IOs, 5353792512 bytes 2024-09-18T01:02:25.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12100 IOs, 5368625152 bytes 2024-09-18T01:02:25.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12200 IOs, 5409575936 bytes 2024-09-18T01:02:25.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12300 IOs, 5494356992 bytes 2024-09-18T01:02:25.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12400 IOs, 5559575552 bytes 2024-09-18T01:02:25.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12500 IOs, 5659639808 bytes 2024-09-18T01:02:25.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:02:25.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:02:26.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T01:02:26.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:26.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:26.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:26.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:02:26.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:02:26.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:02:26.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:02:26.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:02:26.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T01:02:26.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T01:02:26.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:02:26.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+replaying 2024-09-18T01:02:26.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":533623296.0,"last_snapshot_bytes":532117504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621275,"remote_snapshot_timestamp":1726621275,"replay_state":"idle"} 2024-09-18T01:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: service: mirror.1 on smithi150 2024-09-18T01:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:02:03 2024-09-18T01:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T01:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T01:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+stopped 2024-09-18T01:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: local image is primary 2024-09-18T01:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:02:03' 2024-09-18T01:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:02:26.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: state: up+replaying 2024-09-18T01:02:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ description: .*' 2024-09-18T01:02:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":533623296.0,"last_snapshot_bytes":532117504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621275,"remote_snapshot_timestamp":1726621275,"replay_state":"idle"} 2024-09-18T01:02:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:02:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:02:26.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^up+' 2024-09-18T01:02:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:02:26.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:02:26.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: service: mirror.1 on smithi150 2024-09-18T01:02:26.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update 2024-09-18T01:02:26.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:02:26.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:02:03' 2024-09-18T01:02:26.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-18 01:02:03' 2024-09-18T01:02:26.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:02:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local 'prev_last_update=2024-09-18 01:02:03' 2024-09-18T01:02:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:02:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:02:26.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:02:26.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='OKOKOK142134284mirror.1smithi15019.1.1-330-gc2ddf972trueOK42174290mirror.2smithi15019.1.1-330-gc2ddf972falseOK42214285mirror.3smithi15019.1.1-330-gc2ddf972falseOK43444347mirror.0smithi15019.1.1-330-gc2ddf972falseOKtestc32f6c34-fc35-4674-a05b-f4b6c1db7403up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":533623296.0,"last_snapshot_bytes":532117504,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621275,"remote_snapshot_timestamp":1726621275,"replay_state":"idle"}42134284mirror.1smithi1502024-09-18 01:02:03cluster2b28075f7-24f5-441c-873b-85d148cb477bup+stoppedlocal image is primary2024-09-18 01:02:03' 2024-09-18T01:02:26.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update state 2024-09-18T01:02:26.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-18T01:02:26.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:02:03' 2024-09-18T01:02:26.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-18T01:02:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=up+replaying 2024-09-18T01:02:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:02:26.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep up+replaying 2024-09-18T01:02:26.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:02:26.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:02:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:02:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap4 2024-09-18T01:02:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap4 2024-09-18T01:02:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:02:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap4 2024-09-18T01:02:26.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:02:33.360 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9010964762924839) > weight (0.25) 2024-09-18T01:02:33.360 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7790446790167556) > weight (0.25) 2024-09-18T01:02:33.360 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7331538490215762) > weight (0.25) 2024-09-18T01:02:33.361 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.4120873584863295) > weight (0.25) 2024-09-18T01:02:33.361 INFO:tasks.rbd_mirror_thrash:waiting for 66 secs before thrashing 2024-09-18T01:02:48.293 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-18T01:02:48.294 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-18T01:02:48.294 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-09-18T01:02:48.295 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:02:48.294+0000 7f3b7ef88640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 39278) UID: 1000 2024-09-18T01:02:53.295 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-18T01:02:53.295 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:02:53.296 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-18T01:02:53.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-18T01:02:53.296 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-18T01:02:53.300 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-18T01:02:53.300 INFO:tasks.rbd_mirror_thrash:waiting for 96 secs before thrashing 2024-09-18T01:02:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T01:02:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:57.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T01:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:02:57.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:02:58.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T01:02:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:02:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:02:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:02:58.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:02:58.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T01:02:58.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:02:58.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:02:58.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:02:58.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:02:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:02:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:02:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:02:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:02:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:02:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:02:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:02:58.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:02:58.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:02:58.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:02:58.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:02:58.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:02:58.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:02:58.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:02:58.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:02:58.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:02:58.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:02:58.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:02:58.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:02:58.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:02:58.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:02:58.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:02:58.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:02:58.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:02:58.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:02:58.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:02:58.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:02:58.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:02:58.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:02:58.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:02:58.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T01:02:58.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T01:02:58.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:02:58.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:02:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:02:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:02:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:02:58.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:02:58.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:02:58.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:02:58.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:02:58.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:02:58.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:02:58.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:02:58.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:02:58.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T01:02:58.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:02:58.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:02:58.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:02:58.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:02:58.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:02:58.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:02:58.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:02:58.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:02:58.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-18T01:02:58.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:02:58.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:02:58.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:02:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:02:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-18T01:02:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:02:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:02:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-test.status 2024-09-18T01:02:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-test.status 2024-09-18T01:02:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:02:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:02:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-test.status 2024-09-18T01:02:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-test.status 2024-09-18T01:02:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:02:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:02:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-18T01:02:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:02:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:02:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:02:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-18T01:03:00.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 14 2024-09-18T01:03:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:03:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:03:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:03:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-18T01:03:00.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:03:00.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:03:00.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=14 2024-09-18T01:03:00.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:03:00.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:03:00.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:03:00.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:00.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:00.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:00.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:00.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:00.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:03:00.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:03:00.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:03:00.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:03:00.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:03:00.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:03:00.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:03:00.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:03:00.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 14 = '' 2024-09-18T01:03:00.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:03:00.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:03:01.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:01.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:01.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:01.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:01.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:01.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:03:01.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:03:01.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:03:01.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:03:01.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:03:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:03:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:03:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:03:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 14 = '' 2024-09-18T01:03:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:03:01.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:03:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:02.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:02.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:03:02.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:03:02.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:03:02.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:03:02.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:03:02.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:03:02.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:03:02.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:03:02.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 14 = '' 2024-09-18T01:03:02.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:03:02.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:03:03.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:03.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:03.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:03.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:03.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:03.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:03:03.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:03:03.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:03:03.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=14 2024-09-18T01:03:03.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 14 = 14 2024-09-18T01:03:03.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:03.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror test snap4 2024-09-18T01:03:03.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:03.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:03.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:03.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap4 2024-09-18T01:03:03.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:03:03.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:03:03.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:03:04.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/test@snap4 2024-09-18T01:03:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T01:03:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:03:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:03:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 8 2024-09-18T01:03:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc231b3fe3 2024-09-18T01:03:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc231b3fe3 2024-09-18T01:03:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:03:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:03:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:03:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:03:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:03:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:01:21 2024 2024-09-18T01:03:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:03:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:03:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:03:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:03:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:03:04.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:03:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snap3-clean ']' 2024-09-18T01:03:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap3-clean 2024-09-18T01:03:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap3-clean 2024-09-18T01:03:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:03:04.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:03:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:04.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:03:05.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:03:05.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:06.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:03:06.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:06.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap4 2024-09-18T01:03:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap4 2024-09-18T01:03:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:03:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:03:06.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:06.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:06.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:03:07.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:03:07.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:07.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:03:07.815 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:08.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ clean_snap_name=snap4-clean 2024-09-18T01:03:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap4-clean 2024-09-18T01:03:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:03:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:08.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap4-clean 2024-09-18T01:03:08.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20 2024-09-18T01:03:08.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:03:08.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror test 2024-09-18T01:03:08.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:03:08.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:08.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:08.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:03:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=6 2024-09-18T01:03:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:03:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-18T01:03:08.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 25326592 bytes 2024-09-18T01:03:08.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 60581888 bytes 2024-09-18T01:03:08.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 145126400 bytes 2024-09-18T01:03:08.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 210845696 bytes 2024-09-18T01:03:08.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 227364864 bytes 2024-09-18T01:03:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 255211520 bytes 2024-09-18T01:03:09.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 268461056 bytes 2024-09-18T01:03:09.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 356569088 bytes 2024-09-18T01:03:09.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 392317952 bytes 2024-09-18T01:03:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 435941376 bytes 2024-09-18T01:03:09.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 495602688 bytes 2024-09-18T01:03:09.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 529746944 bytes 2024-09-18T01:03:09.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 567490560 bytes 2024-09-18T01:03:09.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 625304576 bytes 2024-09-18T01:03:09.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 680990720 bytes 2024-09-18T01:03:10.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 754668544 bytes 2024-09-18T01:03:10.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 805832704 bytes 2024-09-18T01:03:10.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 826762240 bytes 2024-09-18T01:03:10.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 887790592 bytes 2024-09-18T01:03:10.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 894902272 bytes 2024-09-18T01:03:10.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 950803456 bytes 2024-09-18T01:03:10.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 970364928 bytes 2024-09-18T01:03:10.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1000369152 bytes 2024-09-18T01:03:10.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1038130176 bytes 2024-09-18T01:03:10.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1097760768 bytes 2024-09-18T01:03:10.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1139790848 bytes 2024-09-18T01:03:11.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1221464064 bytes 2024-09-18T01:03:11.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1289420800 bytes 2024-09-18T01:03:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1313309696 bytes 2024-09-18T01:03:11.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1348572160 bytes 2024-09-18T01:03:11.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1443710976 bytes 2024-09-18T01:03:11.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1496639488 bytes 2024-09-18T01:03:11.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1561810944 bytes 2024-09-18T01:03:11.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1595131904 bytes 2024-09-18T01:03:11.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1621979136 bytes 2024-09-18T01:03:12.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1638612992 bytes 2024-09-18T01:03:12.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1647808512 bytes 2024-09-18T01:03:12.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1677891584 bytes 2024-09-18T01:03:12.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1756078080 bytes 2024-09-18T01:03:12.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1824944128 bytes 2024-09-18T01:03:12.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1829293056 bytes 2024-09-18T01:03:12.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1841924096 bytes 2024-09-18T01:03:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1891323904 bytes 2024-09-18T01:03:12.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1941681152 bytes 2024-09-18T01:03:12.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1958368256 bytes 2024-09-18T01:03:12.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2057438208 bytes 2024-09-18T01:03:13.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2131606528 bytes 2024-09-18T01:03:13.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2158828544 bytes 2024-09-18T01:03:13.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2204030976 bytes 2024-09-18T01:03:13.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2324954112 bytes 2024-09-18T01:03:13.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2350848000 bytes 2024-09-18T01:03:13.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2405909504 bytes 2024-09-18T01:03:13.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2435969024 bytes 2024-09-18T01:03:13.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2441460736 bytes 2024-09-18T01:03:13.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2486257664 bytes 2024-09-18T01:03:13.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2501820416 bytes 2024-09-18T01:03:14.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2554695680 bytes 2024-09-18T01:03:14.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2570265600 bytes 2024-09-18T01:03:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:03:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:03:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:03:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-18T01:03:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:03:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:03:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:03:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:03:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:03:15.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T01:03:15.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:15.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:15.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:15.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:03:15.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:03:15.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:03:15.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:03:15.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:03:15.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T01:03:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T01:03:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:03:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+replaying 2024-09-18T01:03:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: replaying, {"bytes_per_second":6011835.73,"bytes_per_snapshot":534430720.0,"last_snapshot_bytes":534743040,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621379,"remote_snapshot_timestamp":1726621379,"replay_state":"idle"} 2024-09-18T01:03:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: service: mirror.1 on smithi150 2024-09-18T01:03:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:03:03 2024-09-18T01:03:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T01:03:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T01:03:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+stopped 2024-09-18T01:03:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: local image is primary 2024-09-18T01:03:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:03:03' 2024-09-18T01:03:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:03:15.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: state: up+replaying 2024-09-18T01:03:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ description: .*' 2024-09-18T01:03:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: description: replaying, {"bytes_per_second":6011835.73,"bytes_per_snapshot":534430720.0,"last_snapshot_bytes":534743040,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621379,"remote_snapshot_timestamp":1726621379,"replay_state":"idle"} 2024-09-18T01:03:15.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:03:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:03:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^up+' 2024-09-18T01:03:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:03:15.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:03:15.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: service: mirror.1 on smithi150 2024-09-18T01:03:15.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update 2024-09-18T01:03:15.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:03:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:03:03' 2024-09-18T01:03:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-18 01:03:03' 2024-09-18T01:03:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:03:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local 'prev_last_update=2024-09-18 01:03:03' 2024-09-18T01:03:15.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:03:15.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:03:15.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:03:15.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='OKOKOK142134284mirror.1smithi15019.1.1-330-gc2ddf972trueOK42174290mirror.2smithi15019.1.1-330-gc2ddf972falseOK42214285mirror.3smithi15019.1.1-330-gc2ddf972falseOK44134416mirror.0smithi15019.1.1-330-gc2ddf972falseOKtestc32f6c34-fc35-4674-a05b-f4b6c1db7403up+replayingreplaying, {"bytes_per_second":6011835.73,"bytes_per_snapshot":534430720.0,"last_snapshot_bytes":534743040,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621379,"remote_snapshot_timestamp":1726621379,"replay_state":"idle"}42134284mirror.1smithi1502024-09-18 01:03:03cluster2b28075f7-24f5-441c-873b-85d148cb477bup+stoppedlocal image is primary2024-09-18 01:03:03' 2024-09-18T01:03:15.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update state 2024-09-18T01:03:15.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-18T01:03:15.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:03:03' 2024-09-18T01:03:15.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-18T01:03:15.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=up+replaying 2024-09-18T01:03:15.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:03:15.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep up+replaying 2024-09-18T01:03:15.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:03:15.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:15.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:15.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap5 2024-09-18T01:03:15.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap5 2024-09-18T01:03:15.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:03:15.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:15.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:15.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap5 2024-09-18T01:03:15.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:03:39.362 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-18T01:03:39.363 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-18T01:03:39.363 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-18T01:03:39.364 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:03:39.363+0000 7fdeb4638640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 39537) UID: 1000 2024-09-18T01:03:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T01:03:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T01:03:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:03:44.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:03:44.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:03:44.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:03:45.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T01:03:45.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:45.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:45.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:45.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:03:45.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:03:45.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:03:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T01:03:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:03:45.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:03:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:03:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:03:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:03:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:03:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:03:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:03:45.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:03:45.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:03:45.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:03:45.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T01:03:45.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:03:45.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:03:45.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:03:45.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:03:45.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:03:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:03:45.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:03:45.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:03:45.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:03:45.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:03:45.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T01:03:45.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T01:03:45.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:03:45.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:03:45.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:03:45.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T01:03:45.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:03:45.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:03:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:03:45.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:03:45.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:03:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:03:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:03:45.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T01:03:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:03:46.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:03:46.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:03:46.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:03:46.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-18T01:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-18T01:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-test.status 2024-09-18T01:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-test.status 2024-09-18T01:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:03:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:03:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-test.status 2024-09-18T01:03:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-test.status 2024-09-18T01:03:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:03:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-18T01:03:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:03:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-18T01:03:46.364 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-18T01:03:46.364 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:03:46.364 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-18T01:03:46.364 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-18T01:03:46.365 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-18T01:03:46.368 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-18T01:03:46.368 INFO:tasks.rbd_mirror_thrash:waiting for 101 secs before thrashing 2024-09-18T01:03:47.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 17 2024-09-18T01:03:47.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:03:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:03:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:03:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-18T01:03:47.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:03:47.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:03:47.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=17 2024-09-18T01:03:47.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:03:47.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:03:47.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:03:48.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:03:48.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:03:48.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:03:48.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:03:48.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:03:48.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:03:48.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:03:48.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 17 = '' 2024-09-18T01:03:48.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:03:48.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:03:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:48.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:03:48.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:03:48.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:03:48.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:03:48.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:03:48.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:03:48.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:03:48.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:03:48.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 17 = '' 2024-09-18T01:03:48.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:03:48.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:03:49.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:03:49.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:03:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:03:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:03:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:03:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:03:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:03:49.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:03:49.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 17 = '' 2024-09-18T01:03:49.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:03:49.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:03:51.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:51.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:51.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:51.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:51.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:03:51.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:03:51.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:03:51.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:03:51.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=17 2024-09-18T01:03:51.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 17 = 17 2024-09-18T01:03:51.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:51.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror test snap5 2024-09-18T01:03:51.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:03:51.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:51.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:51.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap5 2024-09-18T01:03:51.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:03:51.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:03:51.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:03:52.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/test@snap5 2024-09-18T01:03:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T01:03:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:03:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:03:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 10 2024-09-18T01:03:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc231b3fe3 2024-09-18T01:03:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc231b3fe3 2024-09-18T01:03:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:03:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:03:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:03:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:03:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:03:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:03:05 2024 2024-09-18T01:03:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:03:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:03:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:03:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:03:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:03:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:03:52.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:52.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snap4-clean ']' 2024-09-18T01:03:52.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap4-clean 2024-09-18T01:03:52.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:52.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:52.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap4-clean 2024-09-18T01:03:52.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:03:52.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:03:52.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:52.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:52.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:03:52.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:03:52.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:53.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:03:53.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:53.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap5 2024-09-18T01:03:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap5 2024-09-18T01:03:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:03:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:03:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:53.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:53.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:03:54.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:03:54.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:55.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:03:55.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:55.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:03:55.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:03:55.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ clean_snap_name=snap5-clean 2024-09-18T01:03:55.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap5-clean 2024-09-18T01:03:55.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:03:55.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:55.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:55.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap5-clean 2024-09-18T01:03:55.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20 2024-09-18T01:03:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:03:55.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror test 2024-09-18T01:03:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:03:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:03:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:03:55.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:03:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=9 2024-09-18T01:03:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:03:55.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-18T01:03:56.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 35241984 bytes 2024-09-18T01:03:56.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 119412736 bytes 2024-09-18T01:03:56.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 156697600 bytes 2024-09-18T01:03:56.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 167252992 bytes 2024-09-18T01:03:56.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 190883840 bytes 2024-09-18T01:03:56.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 235759616 bytes 2024-09-18T01:03:56.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 258927616 bytes 2024-09-18T01:03:56.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 314188800 bytes 2024-09-18T01:03:56.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 407958528 bytes 2024-09-18T01:03:56.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 466161664 bytes 2024-09-18T01:03:57.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 545200128 bytes 2024-09-18T01:03:57.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 610524160 bytes 2024-09-18T01:03:57.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 671375360 bytes 2024-09-18T01:03:57.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 749797376 bytes 2024-09-18T01:03:57.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 772960256 bytes 2024-09-18T01:03:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 851163136 bytes 2024-09-18T01:03:57.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 875535360 bytes 2024-09-18T01:03:57.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 927976448 bytes 2024-09-18T01:03:58.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 1035813888 bytes 2024-09-18T01:03:58.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 1079835648 bytes 2024-09-18T01:03:58.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1132152832 bytes 2024-09-18T01:03:58.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1135668224 bytes 2024-09-18T01:03:58.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1203721216 bytes 2024-09-18T01:03:58.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1273357312 bytes 2024-09-18T01:03:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1294048256 bytes 2024-09-18T01:03:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1322160128 bytes 2024-09-18T01:03:58.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1399271424 bytes 2024-09-18T01:03:58.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1414748160 bytes 2024-09-18T01:03:58.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1423575040 bytes 2024-09-18T01:03:59.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1524747264 bytes 2024-09-18T01:03:59.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1560038400 bytes 2024-09-18T01:03:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1591898112 bytes 2024-09-18T01:03:59.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1601768448 bytes 2024-09-18T01:03:59.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1611547648 bytes 2024-09-18T01:03:59.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1643294720 bytes 2024-09-18T01:03:59.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1710685184 bytes 2024-09-18T01:03:59.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1790713856 bytes 2024-09-18T01:03:59.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1836113920 bytes 2024-09-18T01:04:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1917357056 bytes 2024-09-18T01:04:00.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1936104448 bytes 2024-09-18T01:04:00.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1964120064 bytes 2024-09-18T01:04:00.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1989388288 bytes 2024-09-18T01:04:00.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2006958080 bytes 2024-09-18T01:04:00.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2037536768 bytes 2024-09-18T01:04:00.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2070365184 bytes 2024-09-18T01:04:00.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2106135552 bytes 2024-09-18T01:04:00.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2148728832 bytes 2024-09-18T01:04:00.796 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2155617280 bytes 2024-09-18T01:04:00.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2182176768 bytes 2024-09-18T01:04:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2220422144 bytes 2024-09-18T01:04:01.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2257744896 bytes 2024-09-18T01:04:01.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2282301440 bytes 2024-09-18T01:04:01.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2308378624 bytes 2024-09-18T01:04:01.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2338555904 bytes 2024-09-18T01:04:01.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2380945408 bytes 2024-09-18T01:04:01.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2409651200 bytes 2024-09-18T01:04:01.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2477634560 bytes 2024-09-18T01:04:01.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2498295808 bytes 2024-09-18T01:04:01.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2543178752 bytes 2024-09-18T01:04:01.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2602898432 bytes 2024-09-18T01:04:02.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2642660352 bytes 2024-09-18T01:04:02.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2702132224 bytes 2024-09-18T01:04:02.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2727748608 bytes 2024-09-18T01:04:02.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2742968320 bytes 2024-09-18T01:04:02.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2778433536 bytes 2024-09-18T01:04:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2805379072 bytes 2024-09-18T01:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2875040768 bytes 2024-09-18T01:04:02.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2944575488 bytes 2024-09-18T01:04:02.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3018035200 bytes 2024-09-18T01:04:02.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3051175936 bytes 2024-09-18T01:04:03.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3074902016 bytes 2024-09-18T01:04:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3128457216 bytes 2024-09-18T01:04:03.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3174477824 bytes 2024-09-18T01:04:03.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3204048896 bytes 2024-09-18T01:04:03.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3211806720 bytes 2024-09-18T01:04:03.548 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3254517760 bytes 2024-09-18T01:04:03.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3276916736 bytes 2024-09-18T01:04:03.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3308613632 bytes 2024-09-18T01:04:03.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3373922304 bytes 2024-09-18T01:04:04.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3476800512 bytes 2024-09-18T01:04:04.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3551634432 bytes 2024-09-18T01:04:04.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3562480640 bytes 2024-09-18T01:04:04.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3596096512 bytes 2024-09-18T01:04:04.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3596735488 bytes 2024-09-18T01:04:04.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3625182208 bytes 2024-09-18T01:04:04.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3652794368 bytes 2024-09-18T01:04:04.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3687428096 bytes 2024-09-18T01:04:04.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3717620736 bytes 2024-09-18T01:04:04.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:04:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:04:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:04:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:04:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-18T01:04:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:04:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:04:04.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:04:04.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:04:04.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:04:05.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T01:04:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:04:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:04:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:04:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:04:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:04:05.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T01:04:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T01:04:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:04:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+replaying 2024-09-18T01:04:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: replaying, {"bytes_per_second":17601843.2,"bytes_per_snapshot":531399168.0,"last_snapshot_bytes":528055296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621426,"remote_snapshot_timestamp":1726621426,"replay_state":"idle"} 2024-09-18T01:04:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: service: mirror.1 on smithi150 2024-09-18T01:04:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:04:03 2024-09-18T01:04:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T01:04:05.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T01:04:05.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+stopped 2024-09-18T01:04:05.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: local image is primary 2024-09-18T01:04:05.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:04:03' 2024-09-18T01:04:05.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:04:05.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: state: up+replaying 2024-09-18T01:04:05.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ description: .*' 2024-09-18T01:04:05.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: description: replaying, {"bytes_per_second":17601843.2,"bytes_per_snapshot":531399168.0,"last_snapshot_bytes":528055296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621426,"remote_snapshot_timestamp":1726621426,"replay_state":"idle"} 2024-09-18T01:04:05.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:04:05.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:04:05.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^up+' 2024-09-18T01:04:05.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:04:05.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:04:05.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: service: mirror.1 on smithi150 2024-09-18T01:04:05.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update 2024-09-18T01:04:05.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:04:05.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:04:03' 2024-09-18T01:04:05.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-18 01:04:03' 2024-09-18T01:04:05.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:05.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:05.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:05.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:04:05.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local 'prev_last_update=2024-09-18 01:04:03' 2024-09-18T01:04:05.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:04:05.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:04:05.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:04:05.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='OKOKOK142134284mirror.1smithi15019.1.1-330-gc2ddf972trueOK42174290mirror.2smithi15019.1.1-330-gc2ddf972falseOK42214285mirror.3smithi15019.1.1-330-gc2ddf972falseOK44134416mirror.0smithi15019.1.1-330-gc2ddf972falseOKtestc32f6c34-fc35-4674-a05b-f4b6c1db7403up+replayingreplaying, {"bytes_per_second":17601843.2,"bytes_per_snapshot":531399168.0,"last_snapshot_bytes":528055296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621426,"remote_snapshot_timestamp":1726621426,"replay_state":"idle"}42134284mirror.1smithi1502024-09-18 01:04:03cluster2b28075f7-24f5-441c-873b-85d148cb477bup+stoppedlocal image is primary2024-09-18 01:04:03' 2024-09-18T01:04:05.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update state 2024-09-18T01:04:05.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-18T01:04:05.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:04:03' 2024-09-18T01:04:05.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-18T01:04:05.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=up+replaying 2024-09-18T01:04:05.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:04:05.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep up+replaying 2024-09-18T01:04:05.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:04:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:04:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:04:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap6 2024-09-18T01:04:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap6 2024-09-18T01:04:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:04:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap6 2024-09-18T01:04:05.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:04:29.301 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7235592893547679) > weight (0.25) 2024-09-18T01:04:29.301 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3250531310174132) > weight (0.25) 2024-09-18T01:04:29.301 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-09-18T01:04:29.302 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-09-18T01:04:29.303 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-18T01:04:29.304 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:04:29.302+0000 7f141907d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 37024) UID: 1000 2024-09-18T01:04:34.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T01:04:34.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:34.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:34.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:34.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T01:04:34.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:34.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:34.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:34.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:04:34.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:04:34.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:04:34.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:04:35.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T01:04:35.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:35.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:35.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:35.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:04:35.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:04:35.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:04:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T01:04:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:04:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:04:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:04:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:04:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:04:35.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:04:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:04:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:04:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:04:35.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:04:35.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:04:35.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T01:04:35.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:04:35.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:04:35.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:04:35.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:04:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:04:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:04:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:04:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:04:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:04:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:04:35.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:04:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:04:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:04:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:04:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:04:35.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:04:35.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:04:35.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:04:35.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:04:35.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T01:04:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T01:04:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:04:35.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:04:35.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:04:35.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T01:04:35.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:04:35.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:04:35.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:04:35.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:04:35.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:04:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:04:35.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:04:35.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:04:35.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:04:35.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T01:04:36.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:04:36.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:04:36.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:04:36.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:04:36.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:04:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:04:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:04:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:04:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-18T01:04:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:04:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:04:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:36.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:04:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:04:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-18T01:04:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:04:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:04:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-test.status 2024-09-18T01:04:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-test.status 2024-09-18T01:04:36.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:04:36.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:04:36.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-test.status 2024-09-18T01:04:36.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-test.status 2024-09-18T01:04:36.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:04:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:04:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-18T01:04:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:04:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-18T01:04:36.303 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-09-18T01:04:36.303 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:04:36.304 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-09-18T01:04:36.304 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-18T01:04:36.304 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-18T01:04:36.308 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-18T01:04:36.308 INFO:tasks.rbd_mirror_thrash:waiting for 100 secs before thrashing 2024-09-18T01:04:37.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 20 2024-09-18T01:04:37.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:04:37.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:04:37.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:37.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:37.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:04:37.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-18T01:04:37.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:04:37.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:04:37.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=20 2024-09-18T01:04:37.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:04:37.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:04:37.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:04:38.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:04:38.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:38.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:38.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:38.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:04:38.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:04:38.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:04:38.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:04:38.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:04:38.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:04:38.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:04:38.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:04:38.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:04:38.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 20 = '' 2024-09-18T01:04:38.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:04:38.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:04:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:04:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:04:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:04:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:04:38.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:04:38.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:04:38.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:04:38.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:04:38.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:04:38.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:04:38.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 20 = '' 2024-09-18T01:04:38.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:04:38.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:04:39.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:04:39.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:39.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:39.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:39.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:04:39.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:04:39.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:04:39.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:04:39.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:04:39.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:04:39.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:04:39.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:04:39.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:04:39.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 20 = '' 2024-09-18T01:04:39.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:04:39.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:04:41.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:04:41.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:41.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:41.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:41.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:04:41.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:04:41.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:04:41.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:04:41.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=20 2024-09-18T01:04:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 20 = 20 2024-09-18T01:04:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:04:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror test snap6 2024-09-18T01:04:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap6 2024-09-18T01:04:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:04:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:04:41.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:04:42.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/test@snap6 2024-09-18T01:04:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T01:04:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:04:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:04:42.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 12 2024-09-18T01:04:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc231b3fe3 2024-09-18T01:04:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc231b3fe3 2024-09-18T01:04:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:04:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:04:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:04:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:04:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:04:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:03:05 2024 2024-09-18T01:04:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:04:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:04:42.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:04:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:04:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:04:42.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:04:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:04:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snap5-clean ']' 2024-09-18T01:04:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap5-clean 2024-09-18T01:04:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap5-clean 2024-09-18T01:04:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:04:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:04:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:04:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:04:42.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:04:42.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:04:42.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:04:43.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:04:43.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:04:43.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:04:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:04:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap6 2024-09-18T01:04:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap6 2024-09-18T01:04:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:04:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:04:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:04:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:04:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:04:44.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:04:44.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:04:45.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:04:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:04:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:04:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:04:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ clean_snap_name=snap6-clean 2024-09-18T01:04:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap6-clean 2024-09-18T01:04:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:04:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap6-clean 2024-09-18T01:04:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20 2024-09-18T01:04:45.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:04:45.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror test 2024-09-18T01:04:45.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:04:45.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:45.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:45.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:04:45.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=12 2024-09-18T01:04:45.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:04:45.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-18T01:04:46.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 64475136 bytes 2024-09-18T01:04:46.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 102871040 bytes 2024-09-18T01:04:46.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 125144064 bytes 2024-09-18T01:04:46.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 146435072 bytes 2024-09-18T01:04:46.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 156318720 bytes 2024-09-18T01:04:46.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 178512896 bytes 2024-09-18T01:04:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 228256768 bytes 2024-09-18T01:04:46.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 353281024 bytes 2024-09-18T01:04:46.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 406913024 bytes 2024-09-18T01:04:47.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 498411520 bytes 2024-09-18T01:04:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 517121024 bytes 2024-09-18T01:04:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 588217344 bytes 2024-09-18T01:04:47.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 625158144 bytes 2024-09-18T01:04:47.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 667320320 bytes 2024-09-18T01:04:47.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 715493376 bytes 2024-09-18T01:04:47.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 789506048 bytes 2024-09-18T01:04:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 838759424 bytes 2024-09-18T01:04:47.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 842094592 bytes 2024-09-18T01:04:47.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 859016192 bytes 2024-09-18T01:04:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 886891520 bytes 2024-09-18T01:04:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 906761216 bytes 2024-09-18T01:04:48.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 922286080 bytes 2024-09-18T01:04:48.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 978672640 bytes 2024-09-18T01:04:48.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1009705984 bytes 2024-09-18T01:04:48.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1074699264 bytes 2024-09-18T01:04:48.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1110706176 bytes 2024-09-18T01:04:48.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1141095424 bytes 2024-09-18T01:04:48.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1211624448 bytes 2024-09-18T01:04:48.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1261275136 bytes 2024-09-18T01:04:49.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1291307008 bytes 2024-09-18T01:04:49.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1368764416 bytes 2024-09-18T01:04:49.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1390244864 bytes 2024-09-18T01:04:49.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1443140608 bytes 2024-09-18T01:04:49.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1459769344 bytes 2024-09-18T01:04:49.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1466008576 bytes 2024-09-18T01:04:49.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1468904448 bytes 2024-09-18T01:04:49.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1606738944 bytes 2024-09-18T01:04:49.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1619068928 bytes 2024-09-18T01:04:49.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1639375872 bytes 2024-09-18T01:04:50.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1655916544 bytes 2024-09-18T01:04:50.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1772451840 bytes 2024-09-18T01:04:50.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1815171072 bytes 2024-09-18T01:04:50.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1817739264 bytes 2024-09-18T01:04:50.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1922466816 bytes 2024-09-18T01:04:50.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1948371968 bytes 2024-09-18T01:04:50.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2008040448 bytes 2024-09-18T01:04:50.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2055243776 bytes 2024-09-18T01:04:50.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2076098560 bytes 2024-09-18T01:04:51.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2140494848 bytes 2024-09-18T01:04:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2195225600 bytes 2024-09-18T01:04:51.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2248215552 bytes 2024-09-18T01:04:51.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2274542592 bytes 2024-09-18T01:04:51.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2319165440 bytes 2024-09-18T01:04:51.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2344718336 bytes 2024-09-18T01:04:51.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2391692288 bytes 2024-09-18T01:04:51.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2430238720 bytes 2024-09-18T01:04:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2503302144 bytes 2024-09-18T01:04:51.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2564496384 bytes 2024-09-18T01:04:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2631561216 bytes 2024-09-18T01:04:52.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2680466432 bytes 2024-09-18T01:04:52.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2731240448 bytes 2024-09-18T01:04:52.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2844045312 bytes 2024-09-18T01:04:52.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2849079296 bytes 2024-09-18T01:04:52.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2886259712 bytes 2024-09-18T01:04:52.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2925715456 bytes 2024-09-18T01:04:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2975962112 bytes 2024-09-18T01:04:52.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3003395072 bytes 2024-09-18T01:04:53.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3073714176 bytes 2024-09-18T01:04:53.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3112730624 bytes 2024-09-18T01:04:53.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3181341696 bytes 2024-09-18T01:04:53.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3186764800 bytes 2024-09-18T01:04:53.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3198461952 bytes 2024-09-18T01:04:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3260751872 bytes 2024-09-18T01:04:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3294331904 bytes 2024-09-18T01:04:53.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3338312704 bytes 2024-09-18T01:04:53.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3358785536 bytes 2024-09-18T01:04:53.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3424991232 bytes 2024-09-18T01:04:54.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3447177216 bytes 2024-09-18T01:04:54.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3470578688 bytes 2024-09-18T01:04:54.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3503668224 bytes 2024-09-18T01:04:54.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3507479552 bytes 2024-09-18T01:04:54.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3546039296 bytes 2024-09-18T01:04:54.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3578331136 bytes 2024-09-18T01:04:54.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3645942784 bytes 2024-09-18T01:04:54.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3706667008 bytes 2024-09-18T01:04:54.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3725157376 bytes 2024-09-18T01:04:54.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3786727424 bytes 2024-09-18T01:04:55.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3822152704 bytes 2024-09-18T01:04:55.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3837375488 bytes 2024-09-18T01:04:55.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3842120704 bytes 2024-09-18T01:04:55.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3861414912 bytes 2024-09-18T01:04:55.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3972035584 bytes 2024-09-18T01:04:55.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4007786496 bytes 2024-09-18T01:04:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4088728576 bytes 2024-09-18T01:04:55.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4138426368 bytes 2024-09-18T01:04:55.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4165769216 bytes 2024-09-18T01:04:56.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4216947712 bytes 2024-09-18T01:04:56.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4227796992 bytes 2024-09-18T01:04:56.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4247648256 bytes 2024-09-18T01:04:56.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4264236032 bytes 2024-09-18T01:04:56.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4266580992 bytes 2024-09-18T01:04:56.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4289777664 bytes 2024-09-18T01:04:56.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4316974080 bytes 2024-09-18T01:04:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4371769344 bytes 2024-09-18T01:04:56.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4429893632 bytes 2024-09-18T01:04:56.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4507587584 bytes 2024-09-18T01:04:56.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4536718336 bytes 2024-09-18T01:04:57.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4557088768 bytes 2024-09-18T01:04:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4582414336 bytes 2024-09-18T01:04:57.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 4620273664 bytes 2024-09-18T01:04:57.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11100 IOs, 4631995392 bytes 2024-09-18T01:04:57.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11200 IOs, 4664331264 bytes 2024-09-18T01:04:57.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11300 IOs, 4697268224 bytes 2024-09-18T01:04:57.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11400 IOs, 4789244928 bytes 2024-09-18T01:04:57.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11500 IOs, 4808550400 bytes 2024-09-18T01:04:57.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:04:57.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:04:57.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:04:57.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:04:57.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-18T01:04:57.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:04:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:04:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:04:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:04:57.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:04:58.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T01:04:58.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:58.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:58.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:58.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:04:58.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:04:58.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:04:58.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:04:58.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:04:58.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T01:04:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T01:04:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:04:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+replaying 2024-09-18T01:04:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":531399168.0,"last_snapshot_bytes":528055296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621426,"remote_snapshot_timestamp":1726621426,"replay_state":"idle"} 2024-09-18T01:04:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: service: mirror.1 on smithi150 2024-09-18T01:04:58.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:04:33 2024-09-18T01:04:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T01:04:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T01:04:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+stopped 2024-09-18T01:04:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: local image is primary 2024-09-18T01:04:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:04:33' 2024-09-18T01:04:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:04:58.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: state: up+replaying 2024-09-18T01:04:58.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ description: .*' 2024-09-18T01:04:58.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":531399168.0,"last_snapshot_bytes":528055296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621426,"remote_snapshot_timestamp":1726621426,"replay_state":"idle"} 2024-09-18T01:04:58.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:04:58.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:04:58.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^up+' 2024-09-18T01:04:58.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:04:58.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:04:58.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: service: mirror.1 on smithi150 2024-09-18T01:04:58.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update 2024-09-18T01:04:58.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:04:58.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:04:33' 2024-09-18T01:04:58.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-18 01:04:33' 2024-09-18T01:04:58.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:04:58.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:58.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:58.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:04:58.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local 'prev_last_update=2024-09-18 01:04:33' 2024-09-18T01:04:58.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:04:58.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:04:58.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:04:58.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='OKOKOK142134284mirror.1smithi15019.1.1-330-gc2ddf972trueOK42214285mirror.3smithi15019.1.1-330-gc2ddf972falseOK44134416mirror.0smithi15019.1.1-330-gc2ddf972falseOK44814484mirror.2smithi15019.1.1-330-gc2ddf972falseOKtestc32f6c34-fc35-4674-a05b-f4b6c1db7403up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":531399168.0,"last_snapshot_bytes":528055296,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621426,"remote_snapshot_timestamp":1726621426,"replay_state":"idle"}42134284mirror.1smithi1502024-09-18 01:04:33cluster2b28075f7-24f5-441c-873b-85d148cb477bup+stoppedlocal image is primary2024-09-18 01:04:33' 2024-09-18T01:04:58.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update state 2024-09-18T01:04:58.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-18T01:04:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:04:33' 2024-09-18T01:04:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-18T01:04:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=up+replaying 2024-09-18T01:04:58.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:04:58.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep up+replaying 2024-09-18T01:04:58.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:04:58.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:04:58.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:04:58.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap7 2024-09-18T01:04:58.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap7 2024-09-18T01:04:58.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:04:58.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:04:58.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:04:58.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap7 2024-09-18T01:04:58.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:05:27.369 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3981027374792794) > weight (0.25) 2024-09-18T01:05:27.370 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.2747937540437857) > weight (0.25) 2024-09-18T01:05:27.370 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5819238061328692) > weight (0.25) 2024-09-18T01:05:27.370 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.701860802503503) > weight (0.25) 2024-09-18T01:05:27.370 INFO:tasks.rbd_mirror_thrash:waiting for 108 secs before thrashing 2024-09-18T01:05:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T01:05:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:05:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T01:05:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:05:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:05:27.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:05:27.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:27.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:05:28.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T01:05:28.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:05:28.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:28.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:28.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:05:28.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:05:28.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:05:28.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T01:05:28.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:05:28.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:05:28.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:05:28.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:05:28.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:05:28.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:05:28.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:28.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:05:28.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:05:28.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:05:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:05:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T01:05:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:05:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:05:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:05:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:05:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:05:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:05:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:05:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:05:28.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:05:28.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:05:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:05:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:05:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:05:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:05:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:05:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:05:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:05:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:05:28.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:05:28.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:05:28.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:05:28.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T01:05:28.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T01:05:28.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:05:28.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:05:28.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:05:28.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T01:05:28.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:05:28.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:05:28.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:05:28.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:05:28.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:05:29.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:05:29.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:05:29.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:05:29.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T01:05:29.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:05:29.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:05:29.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:05:29.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:05:29.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:05:29.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:05:29.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-test.status 2024-09-18T01:05:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-test.status 2024-09-18T01:05:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:05:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:05:29.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-test.status 2024-09-18T01:05:29.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-test.status 2024-09-18T01:05:29.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:05:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:05:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-18T01:05:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:05:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-18T01:05:30.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 23 2024-09-18T01:05:30.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:05:30.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:05:30.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:30.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:30.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:05:30.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-18T01:05:30.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:05:30.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:05:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=23 2024-09-18T01:05:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:05:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:05:30.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:05:31.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:05:31.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:05:31.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:31.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:31.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:05:31.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:05:31.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:05:31.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:05:31.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:05:31.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:05:31.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:05:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:05:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:05:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 23 = '' 2024-09-18T01:05:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:05:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:05:31.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:05:31.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:05:31.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:31.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:31.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:05:31.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:05:31.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:05:31.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:05:31.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:05:31.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:05:31.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:05:31.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:05:31.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:05:31.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 23 = '' 2024-09-18T01:05:31.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:05:31.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:05:32.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:05:32.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:05:32.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:32.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:32.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:05:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:05:32.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:05:32.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:05:32.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:05:32.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:05:32.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:05:32.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:05:32.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:05:32.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 23 = '' 2024-09-18T01:05:32.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:05:32.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:05:34.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:05:34.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:05:34.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:34.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:34.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:05:34.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:05:34.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:05:34.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:05:34.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=23 2024-09-18T01:05:34.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 23 = 23 2024-09-18T01:05:34.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:05:34.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror test snap7 2024-09-18T01:05:34.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:05:34.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:34.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:34.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap7 2024-09-18T01:05:34.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:05:34.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:05:34.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:05:35.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/test@snap7 2024-09-18T01:05:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T01:05:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:05:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:05:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 14 2024-09-18T01:05:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc231b3fe3 2024-09-18T01:05:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc231b3fe3 2024-09-18T01:05:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:05:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:05:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:05:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:05:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:05:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:04:42 2024 2024-09-18T01:05:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:05:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:05:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:05:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:05:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:05:35.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:05:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:05:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snap6-clean ']' 2024-09-18T01:05:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap6-clean 2024-09-18T01:05:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:35.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap6-clean 2024-09-18T01:05:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:05:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:05:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:05:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:05:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:05:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:05:35.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:05:36.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:05:36.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:05:36.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:05:36.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:05:36.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap7 2024-09-18T01:05:36.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:36.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:36.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap7 2024-09-18T01:05:36.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:05:36.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:05:36.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:05:36.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:05:36.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:05:37.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:05:37.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:05:38.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:05:38.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:05:38.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:05:38.406 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:05:38.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ clean_snap_name=snap7-clean 2024-09-18T01:05:38.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap7-clean 2024-09-18T01:05:38.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:05:38.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:38.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:38.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap7-clean 2024-09-18T01:05:38.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20 2024-09-18T01:05:38.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:05:38.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror test 2024-09-18T01:05:38.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:05:38.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:38.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:38.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:05:38.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=12 2024-09-18T01:05:38.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:05:38.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-18T01:05:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 631808 bytes 2024-09-18T01:05:39.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 46960640 bytes 2024-09-18T01:05:39.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 56451072 bytes 2024-09-18T01:05:39.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 81052672 bytes 2024-09-18T01:05:39.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 100959232 bytes 2024-09-18T01:05:39.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 118434816 bytes 2024-09-18T01:05:39.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 134419456 bytes 2024-09-18T01:05:39.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 154633216 bytes 2024-09-18T01:05:39.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 193736704 bytes 2024-09-18T01:05:39.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 194341888 bytes 2024-09-18T01:05:39.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 197466112 bytes 2024-09-18T01:05:40.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 241965056 bytes 2024-09-18T01:05:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 312223744 bytes 2024-09-18T01:05:40.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 371099648 bytes 2024-09-18T01:05:40.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 423431168 bytes 2024-09-18T01:05:40.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 466995200 bytes 2024-09-18T01:05:40.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 519576576 bytes 2024-09-18T01:05:40.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 520359936 bytes 2024-09-18T01:05:40.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 575745024 bytes 2024-09-18T01:05:40.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 593238016 bytes 2024-09-18T01:05:40.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 638452736 bytes 2024-09-18T01:05:41.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 662194176 bytes 2024-09-18T01:05:41.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 712307712 bytes 2024-09-18T01:05:41.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 737518592 bytes 2024-09-18T01:05:41.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 814995456 bytes 2024-09-18T01:05:41.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 815624192 bytes 2024-09-18T01:05:41.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 836079616 bytes 2024-09-18T01:05:41.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 880680960 bytes 2024-09-18T01:05:41.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 903979008 bytes 2024-09-18T01:05:41.908 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 982334464 bytes 2024-09-18T01:05:42.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1035096064 bytes 2024-09-18T01:05:42.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1047482368 bytes 2024-09-18T01:05:42.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1073500160 bytes 2024-09-18T01:05:42.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1132849152 bytes 2024-09-18T01:05:42.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1160685568 bytes 2024-09-18T01:05:42.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1194032128 bytes 2024-09-18T01:05:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1228118016 bytes 2024-09-18T01:05:42.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1249742848 bytes 2024-09-18T01:05:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1361761280 bytes 2024-09-18T01:05:43.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1420055552 bytes 2024-09-18T01:05:43.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1490115584 bytes 2024-09-18T01:05:43.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1501790208 bytes 2024-09-18T01:05:43.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1579295744 bytes 2024-09-18T01:05:43.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1671460864 bytes 2024-09-18T01:05:43.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1728159744 bytes 2024-09-18T01:05:43.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1829620736 bytes 2024-09-18T01:05:43.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1911502848 bytes 2024-09-18T01:05:44.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1987075072 bytes 2024-09-18T01:05:44.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2033476608 bytes 2024-09-18T01:05:44.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2077669376 bytes 2024-09-18T01:05:44.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2095965184 bytes 2024-09-18T01:05:44.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2124689408 bytes 2024-09-18T01:05:44.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2168482816 bytes 2024-09-18T01:05:44.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2181601280 bytes 2024-09-18T01:05:44.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2197448704 bytes 2024-09-18T01:05:44.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2247513088 bytes 2024-09-18T01:05:44.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2290403328 bytes 2024-09-18T01:05:45.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2350110720 bytes 2024-09-18T01:05:45.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2405520384 bytes 2024-09-18T01:05:45.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2434734080 bytes 2024-09-18T01:05:45.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2452597760 bytes 2024-09-18T01:05:45.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2476706816 bytes 2024-09-18T01:05:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2540888064 bytes 2024-09-18T01:05:45.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2577155072 bytes 2024-09-18T01:05:45.797 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2619152384 bytes 2024-09-18T01:05:45.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2645252096 bytes 2024-09-18T01:05:45.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2682599424 bytes 2024-09-18T01:05:46.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2741817344 bytes 2024-09-18T01:05:46.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2782033920 bytes 2024-09-18T01:05:46.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2850787328 bytes 2024-09-18T01:05:46.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2911020032 bytes 2024-09-18T01:05:46.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2966644736 bytes 2024-09-18T01:05:46.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3027721216 bytes 2024-09-18T01:05:46.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3108369408 bytes 2024-09-18T01:05:46.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3146300416 bytes 2024-09-18T01:05:47.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3251069952 bytes 2024-09-18T01:05:47.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3280672768 bytes 2024-09-18T01:05:47.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3295246336 bytes 2024-09-18T01:05:47.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3325545472 bytes 2024-09-18T01:05:47.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3391760384 bytes 2024-09-18T01:05:47.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3423939584 bytes 2024-09-18T01:05:47.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3493449728 bytes 2024-09-18T01:05:47.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3547071488 bytes 2024-09-18T01:05:47.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3569707008 bytes 2024-09-18T01:05:48.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3600249856 bytes 2024-09-18T01:05:48.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3633801216 bytes 2024-09-18T01:05:48.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3741717504 bytes 2024-09-18T01:05:48.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3797851136 bytes 2024-09-18T01:05:48.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3850394624 bytes 2024-09-18T01:05:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3863870464 bytes 2024-09-18T01:05:48.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3924204544 bytes 2024-09-18T01:05:48.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3974713344 bytes 2024-09-18T01:05:48.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4006597632 bytes 2024-09-18T01:05:48.982 INFO:tasks.ceph.cluster2.osd.0.smithi150.stderr:2024-09-18T01:05:48.981+0000 7f8980c28640 -1 reset not still connected to 0x55b4e0090a90 2024-09-18T01:05:49.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4046827520 bytes 2024-09-18T01:05:49.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4071342080 bytes 2024-09-18T01:05:49.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4130877440 bytes 2024-09-18T01:05:49.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4201191424 bytes 2024-09-18T01:05:49.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4283234304 bytes 2024-09-18T01:05:49.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4298017792 bytes 2024-09-18T01:05:49.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4350961664 bytes 2024-09-18T01:05:49.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4373113856 bytes 2024-09-18T01:05:49.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4421797888 bytes 2024-09-18T01:05:50.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4481374208 bytes 2024-09-18T01:05:50.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4521382912 bytes 2024-09-18T01:05:50.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4555054080 bytes 2024-09-18T01:05:50.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4600039424 bytes 2024-09-18T01:05:50.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4628864000 bytes 2024-09-18T01:05:50.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4658529280 bytes 2024-09-18T01:05:50.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4681339904 bytes 2024-09-18T01:05:50.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 4703992832 bytes 2024-09-18T01:05:50.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11100 IOs, 4727119872 bytes 2024-09-18T01:05:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:05:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:05:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:05:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:05:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-18T01:05:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:05:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:50.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:50.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:05:50.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:05:50.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:05:50.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:05:50.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:05:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T01:05:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:05:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:51.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:05:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:05:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:05:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:05:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:05:51.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T01:05:51.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T01:05:51.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:05:51.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+replaying 2024-09-18T01:05:51.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: replaying, {"bytes_per_second":7130316.8,"bytes_per_snapshot":535027200.0,"last_snapshot_bytes":536176640,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621476,"remote_snapshot_timestamp":1726621529,"replay_state":"syncing","seconds_until_synced":75,"syncing_percent":100,"syncing_snapshot_timestamp":1726621529} 2024-09-18T01:05:51.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: service: mirror.1 on smithi150 2024-09-18T01:05:51.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:05:33 2024-09-18T01:05:51.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T01:05:51.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T01:05:51.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+stopped 2024-09-18T01:05:51.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: local image is primary 2024-09-18T01:05:51.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:05:33' 2024-09-18T01:05:51.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:05:51.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: state: up+replaying 2024-09-18T01:05:51.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ description: .*' 2024-09-18T01:05:51.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: description: replaying, {"bytes_per_second":7130316.8,"bytes_per_snapshot":535027200.0,"last_snapshot_bytes":536176640,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621476,"remote_snapshot_timestamp":1726621529,"replay_state":"syncing","seconds_until_synced":75,"syncing_percent":100,"syncing_snapshot_timestamp":1726621529} 2024-09-18T01:05:51.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:05:51.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:05:51.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^up+' 2024-09-18T01:05:51.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:05:51.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:05:51.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: service: mirror.1 on smithi150 2024-09-18T01:05:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update 2024-09-18T01:05:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:05:51.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:05:33' 2024-09-18T01:05:51.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-18 01:05:33' 2024-09-18T01:05:51.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:05:51.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:51.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:51.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:05:51.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local 'prev_last_update=2024-09-18 01:05:33' 2024-09-18T01:05:51.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:05:51.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:05:51.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:05:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='OKOKOK142134284mirror.1smithi15019.1.1-330-gc2ddf972trueOK42214285mirror.3smithi15019.1.1-330-gc2ddf972falseOK44134416mirror.0smithi15019.1.1-330-gc2ddf972falseOK44814484mirror.2smithi15019.1.1-330-gc2ddf972falseOKtestc32f6c34-fc35-4674-a05b-f4b6c1db7403up+replayingreplaying, {"bytes_per_second":7130316.8,"bytes_per_snapshot":535027200.0,"last_snapshot_bytes":536176640,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621476,"remote_snapshot_timestamp":1726621529,"replay_state":"syncing","seconds_until_synced":75,"syncing_percent":100,"syncing_snapshot_timestamp":1726621529}42134284mirror.1smithi1502024-09-18 01:05:33cluster2b28075f7-24f5-441c-873b-85d148cb477bup+stoppedlocal image is primary2024-09-18 01:05:33' 2024-09-18T01:05:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update state 2024-09-18T01:05:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-18T01:05:51.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:05:33' 2024-09-18T01:05:51.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-18T01:05:51.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=up+replaying 2024-09-18T01:05:51.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:05:51.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep up+replaying 2024-09-18T01:05:51.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:05:51.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:05:51.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:05:51.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap8 2024-09-18T01:05:51.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap8 2024-09-18T01:05:51.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:05:51.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:05:51.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:05:51.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap8 2024-09-18T01:05:51.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:06:16.309 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.38004865869171933) > weight (0.25) 2024-09-18T01:06:16.310 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7748821525697374) > weight (0.25) 2024-09-18T01:06:16.310 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.3378355306087487) > weight (0.25) 2024-09-18T01:06:16.310 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2024-09-18T01:06:16.311 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2024-09-18T01:06:16.311 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-18T01:06:16.312 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:06:16.310+0000 7f3fd8f0a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 37077) UID: 1000 2024-09-18T01:06:20.312 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2024-09-18T01:06:20.312 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:06:20.312 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2024-09-18T01:06:20.312 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-09-18T01:06:20.313 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-09-18T01:06:20.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-09-18T01:06:20.316 INFO:tasks.rbd_mirror_thrash:waiting for 84 secs before thrashing 2024-09-18T01:06:21.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T01:06:21.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:06:21.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:21.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:21.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T01:06:21.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:06:21.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:21.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:21.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:06:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:06:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:06:21.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:06:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T01:06:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:06:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:06:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:06:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:06:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T01:06:22.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:06:22.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:06:22.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:06:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:06:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:06:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:06:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:06:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:06:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:06:22.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:06:22.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:06:22.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T01:06:22.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:06:22.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:06:22.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:06:22.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:06:22.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:06:22.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:06:22.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:06:22.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:06:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:06:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:06:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:06:22.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:06:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:06:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:06:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:06:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:06:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:06:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:06:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:06:22.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:06:22.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:06:22.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:06:22.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T01:06:22.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T01:06:22.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:06:22.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:06:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:06:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T01:06:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:06:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:06:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:06:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:06:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:06:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:06:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:06:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:06:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:06:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:06:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:06:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:06:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:06:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:06:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:06:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:06:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:06:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:06:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:06:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:06:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:06:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:06:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:06:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:06:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:06:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:06:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:06:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:06:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:06:22.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:06:22.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:06:22.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:06:22.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:06:22.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T01:06:23.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:06:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:06:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:06:23.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:06:23.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:06:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:06:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:06:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:06:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-18T01:06:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:06:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-test.status 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-test.status 2024-09-18T01:06:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:06:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:06:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-test.status 2024-09-18T01:06:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-test.status 2024-09-18T01:06:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:06:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:06:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-18T01:06:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:06:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-18T01:06:24.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 26 2024-09-18T01:06:24.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:06:24.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:06:24.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:24.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:24.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:06:24.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-18T01:06:24.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:06:24.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:06:24.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=26 2024-09-18T01:06:24.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:06:24.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:06:24.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:06:25.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:06:25.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:06:25.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:25.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:25.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:06:25.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:06:25.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:06:25.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:06:25.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:06:25.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:06:25.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:06:25.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:06:25.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:06:25.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 26 = '' 2024-09-18T01:06:25.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:06:25.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:06:25.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:06:25.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:06:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:06:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:06:25.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:06:25.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:06:25.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:06:25.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:06:25.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:06:25.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:06:25.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:06:25.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 26 = '' 2024-09-18T01:06:25.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:06:25.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:06:26.471 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:06:26.471 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:06:26.471 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:06:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:06:26.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:06:26.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:06:26.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:06:26.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:06:26.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:06:26.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:06:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:06:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 26 = '' 2024-09-18T01:06:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:06:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:06:28.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:06:28.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:06:28.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:28.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:28.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:06:28.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:06:28.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:06:28.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=26 2024-09-18T01:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 26 = 26 2024-09-18T01:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror test snap8 2024-09-18T01:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap8 2024-09-18T01:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:06:29.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/test@snap8 2024-09-18T01:06:29.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T01:06:29.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:06:29.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:06:29.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 16 2024-09-18T01:06:29.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc231b3fe3 2024-09-18T01:06:29.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc231b3fe3 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:04:42 2024 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:06:29.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:06:29.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:06:29.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snap7-clean ']' 2024-09-18T01:06:29.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap7-clean 2024-09-18T01:06:29.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:29.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:29.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap7-clean 2024-09-18T01:06:29.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:06:29.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:06:29.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:06:29.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:06:29.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:06:29.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:06:29.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:06:30.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:06:30.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:06:30.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:06:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap8 2024-09-18T01:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap8 2024-09-18T01:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:06:30.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:06:31.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:06:31.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:06:32.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:06:32.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:06:32.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:06:32.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:06:32.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ clean_snap_name=snap8-clean 2024-09-18T01:06:32.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap8-clean 2024-09-18T01:06:32.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:06:32.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:32.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:32.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap8-clean 2024-09-18T01:06:32.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20 2024-09-18T01:06:32.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:06:32.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror test 2024-09-18T01:06:32.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:06:32.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:32.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:32.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:06:32.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:06:32.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:06:32.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-18T01:06:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 24804352 bytes 2024-09-18T01:06:33.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 72344576 bytes 2024-09-18T01:06:33.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 123952128 bytes 2024-09-18T01:06:33.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 200064000 bytes 2024-09-18T01:06:33.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 278563840 bytes 2024-09-18T01:06:33.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 301648896 bytes 2024-09-18T01:06:33.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 324495360 bytes 2024-09-18T01:06:33.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 350551040 bytes 2024-09-18T01:06:34.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 417798144 bytes 2024-09-18T01:06:34.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 470085632 bytes 2024-09-18T01:06:34.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 516772864 bytes 2024-09-18T01:06:34.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 526548992 bytes 2024-09-18T01:06:34.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 531921920 bytes 2024-09-18T01:06:34.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 556585984 bytes 2024-09-18T01:06:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 573813760 bytes 2024-09-18T01:06:34.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 617137152 bytes 2024-09-18T01:06:34.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 639188992 bytes 2024-09-18T01:06:34.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 639823872 bytes 2024-09-18T01:06:34.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 661307392 bytes 2024-09-18T01:06:35.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 704139264 bytes 2024-09-18T01:06:35.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 748129280 bytes 2024-09-18T01:06:35.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 790992896 bytes 2024-09-18T01:06:35.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 800536576 bytes 2024-09-18T01:06:35.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 902991872 bytes 2024-09-18T01:06:35.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 939599872 bytes 2024-09-18T01:06:35.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 972932096 bytes 2024-09-18T01:06:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1013795840 bytes 2024-09-18T01:06:35.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1017434112 bytes 2024-09-18T01:06:35.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1018094592 bytes 2024-09-18T01:06:36.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1046373376 bytes 2024-09-18T01:06:36.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1058523136 bytes 2024-09-18T01:06:36.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1096048640 bytes 2024-09-18T01:06:36.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1133565952 bytes 2024-09-18T01:06:36.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1229805568 bytes 2024-09-18T01:06:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1230758912 bytes 2024-09-18T01:06:36.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1246211072 bytes 2024-09-18T01:06:36.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1256517632 bytes 2024-09-18T01:06:36.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1326252032 bytes 2024-09-18T01:06:36.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1352922112 bytes 2024-09-18T01:06:37.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1403133952 bytes 2024-09-18T01:06:37.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1448675328 bytes 2024-09-18T01:06:37.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1502211072 bytes 2024-09-18T01:06:37.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1559037952 bytes 2024-09-18T01:06:37.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1600189440 bytes 2024-09-18T01:06:37.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1655749632 bytes 2024-09-18T01:06:37.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1666503680 bytes 2024-09-18T01:06:37.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1690755072 bytes 2024-09-18T01:06:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1707334656 bytes 2024-09-18T01:06:37.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1726919680 bytes 2024-09-18T01:06:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1772206080 bytes 2024-09-18T01:06:38.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 1799277568 bytes 2024-09-18T01:06:38.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 1801851904 bytes 2024-09-18T01:06:38.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 1869107200 bytes 2024-09-18T01:06:38.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 1920843776 bytes 2024-09-18T01:06:38.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 1959043072 bytes 2024-09-18T01:06:38.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 1993594880 bytes 2024-09-18T01:06:38.796 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2002136064 bytes 2024-09-18T01:06:38.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2020514816 bytes 2024-09-18T01:06:38.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2053021696 bytes 2024-09-18T01:06:39.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2075828224 bytes 2024-09-18T01:06:39.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2127405056 bytes 2024-09-18T01:06:39.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2193645568 bytes 2024-09-18T01:06:39.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2231958528 bytes 2024-09-18T01:06:39.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2260829184 bytes 2024-09-18T01:06:39.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2262865920 bytes 2024-09-18T01:06:39.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2284774400 bytes 2024-09-18T01:06:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2319648768 bytes 2024-09-18T01:06:39.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2438165504 bytes 2024-09-18T01:06:40.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2505176064 bytes 2024-09-18T01:06:40.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2521190400 bytes 2024-09-18T01:06:40.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2573884416 bytes 2024-09-18T01:06:40.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2619860992 bytes 2024-09-18T01:06:40.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 2734222336 bytes 2024-09-18T01:06:40.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 2805359616 bytes 2024-09-18T01:06:40.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 2858236928 bytes 2024-09-18T01:06:40.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 2906571776 bytes 2024-09-18T01:06:41.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 2907205632 bytes 2024-09-18T01:06:41.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 2961064960 bytes 2024-09-18T01:06:41.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 2997910528 bytes 2024-09-18T01:06:41.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3044505600 bytes 2024-09-18T01:06:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3078472704 bytes 2024-09-18T01:06:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3132652544 bytes 2024-09-18T01:06:41.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3220442112 bytes 2024-09-18T01:06:41.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3280540672 bytes 2024-09-18T01:06:42.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3345418240 bytes 2024-09-18T01:06:42.151 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3410705408 bytes 2024-09-18T01:06:42.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3482981376 bytes 2024-09-18T01:06:42.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3548438528 bytes 2024-09-18T01:06:42.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3607275520 bytes 2024-09-18T01:06:42.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3660534784 bytes 2024-09-18T01:06:42.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3733267456 bytes 2024-09-18T01:06:42.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3749288960 bytes 2024-09-18T01:06:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3785977856 bytes 2024-09-18T01:06:43.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3799675904 bytes 2024-09-18T01:06:43.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 3825730560 bytes 2024-09-18T01:06:43.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 3831414784 bytes 2024-09-18T01:06:43.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 3856187392 bytes 2024-09-18T01:06:43.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 3865219072 bytes 2024-09-18T01:06:43.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 3886497792 bytes 2024-09-18T01:06:43.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 3937875968 bytes 2024-09-18T01:06:43.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 3963192320 bytes 2024-09-18T01:06:43.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 3987246080 bytes 2024-09-18T01:06:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4041103360 bytes 2024-09-18T01:06:44.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4081168384 bytes 2024-09-18T01:06:44.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4146841600 bytes 2024-09-18T01:06:44.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4159789056 bytes 2024-09-18T01:06:44.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4171327488 bytes 2024-09-18T01:06:44.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4173890560 bytes 2024-09-18T01:06:44.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4231158784 bytes 2024-09-18T01:06:44.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 4286684160 bytes 2024-09-18T01:06:44.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11100 IOs, 4331297792 bytes 2024-09-18T01:06:44.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11200 IOs, 4407537664 bytes 2024-09-18T01:06:45.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11300 IOs, 4440077312 bytes 2024-09-18T01:06:45.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11400 IOs, 4500920320 bytes 2024-09-18T01:06:45.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11500 IOs, 4562853888 bytes 2024-09-18T01:06:45.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11600 IOs, 4615580672 bytes 2024-09-18T01:06:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11700 IOs, 4688945152 bytes 2024-09-18T01:06:45.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11800 IOs, 4776383488 bytes 2024-09-18T01:06:45.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11900 IOs, 4860065792 bytes 2024-09-18T01:06:45.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12000 IOs, 4907162624 bytes 2024-09-18T01:06:46.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12100 IOs, 4946799616 bytes 2024-09-18T01:06:46.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12200 IOs, 5020811264 bytes 2024-09-18T01:06:46.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12300 IOs, 5046825984 bytes 2024-09-18T01:06:46.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12400 IOs, 5138167808 bytes 2024-09-18T01:06:46.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12500 IOs, 5159825408 bytes 2024-09-18T01:06:46.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12600 IOs, 5184505856 bytes 2024-09-18T01:06:46.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12700 IOs, 5219597312 bytes 2024-09-18T01:06:46.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12800 IOs, 5222187008 bytes 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:06:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:06:47.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T01:06:47.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T01:06:47.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T01:06:47.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:06:47.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+replaying 2024-09-18T01:06:47.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: replaying, {"bytes_per_second":17846306.13,"bytes_per_snapshot":535782912.0,"last_snapshot_bytes":535389184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621583,"remote_snapshot_timestamp":1726621583,"replay_state":"idle"} 2024-09-18T01:06:47.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: service: mirror.1 on smithi150 2024-09-18T01:06:47.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:06:33 2024-09-18T01:06:47.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T01:06:47.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T01:06:47.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+stopped 2024-09-18T01:06:47.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: local image is primary 2024-09-18T01:06:47.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:06:33' 2024-09-18T01:06:47.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:06:47.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: state: up+replaying 2024-09-18T01:06:47.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ description: .*' 2024-09-18T01:06:47.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: description: replaying, {"bytes_per_second":17846306.13,"bytes_per_snapshot":535782912.0,"last_snapshot_bytes":535389184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621583,"remote_snapshot_timestamp":1726621583,"replay_state":"idle"} 2024-09-18T01:06:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:06:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:06:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^up+' 2024-09-18T01:06:47.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:06:47.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:06:47.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: service: mirror.1 on smithi150 2024-09-18T01:06:47.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update 2024-09-18T01:06:47.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:06:47.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:06:33' 2024-09-18T01:06:47.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-18 01:06:33' 2024-09-18T01:06:47.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:06:47.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:47.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:47.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:06:47.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local 'prev_last_update=2024-09-18 01:06:33' 2024-09-18T01:06:47.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:06:47.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:06:47.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:06:48.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='OKOKOK142134284mirror.1smithi15019.1.1-330-gc2ddf972trueOK44134416mirror.0smithi15019.1.1-330-gc2ddf972falseOK44814484mirror.2smithi15019.1.1-330-gc2ddf972falseOK45444547mirror.3smithi15019.1.1-330-gc2ddf972falseOKtestc32f6c34-fc35-4674-a05b-f4b6c1db7403up+replayingreplaying, {"bytes_per_second":17846306.13,"bytes_per_snapshot":535782912.0,"last_snapshot_bytes":535389184,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621583,"remote_snapshot_timestamp":1726621583,"replay_state":"idle"}42134284mirror.1smithi1502024-09-18 01:06:33cluster2b28075f7-24f5-441c-873b-85d148cb477bup+stoppedlocal image is primary2024-09-18 01:06:33' 2024-09-18T01:06:48.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update state 2024-09-18T01:06:48.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-18T01:06:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:06:33' 2024-09-18T01:06:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-18T01:06:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=up+replaying 2024-09-18T01:06:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:06:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep up+replaying 2024-09-18T01:06:48.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:06:48.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:06:48.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:06:48.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap9 2024-09-18T01:06:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap9 2024-09-18T01:06:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:06:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:06:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:06:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap9 2024-09-18T01:06:48.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:07:15.371 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-18T01:07:15.372 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-18T01:07:15.372 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-09-18T01:07:15.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:07:15.372+0000 7fd0c5918640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 40821) UID: 1000 2024-09-18T01:07:17.373 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-18T01:07:17.373 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:07:17.374 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-18T01:07:17.374 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-18T01:07:17.374 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-18T01:07:17.377 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-18T01:07:17.377 INFO:tasks.rbd_mirror_thrash:waiting for 74 secs before thrashing 2024-09-18T01:07:18.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T01:07:18.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:07:18.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:18.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:18.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T01:07:18.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:07:18.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:18.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:18.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:07:18.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:07:18.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:18.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:07:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T01:07:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:07:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:07:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:07:19.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:07:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T01:07:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:07:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:07:19.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:07:19.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:07:19.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:07:19.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:07:19.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:19.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:07:19.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:07:19.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:07:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:07:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T01:07:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:07:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:07:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:07:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:07:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:07:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:07:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:07:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:07:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:07:19.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:07:19.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:07:19.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:07:19.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:07:19.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:07:19.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:07:19.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:07:19.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:07:19.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:07:19.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:07:19.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:07:19.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:07:19.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:07:19.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:07:19.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:07:19.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:07:19.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:07:19.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:07:19.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:07:19.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:07:19.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:07:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:07:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:07:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:07:19.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T01:07:19.396 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T01:07:19.396 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:07:19.396 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:07:19.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:07:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:07:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:07:19.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:07:19.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:07:19.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:07:19.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:07:19.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:07:19.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:07:19.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:07:19.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:07:19.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:07:19.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T01:07:19.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:07:19.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:07:19.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:07:19.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:07:19.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:07:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:07:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:07:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:07:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-18T01:07:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:07:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:07:19.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-test.status 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-test.status 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:07:19.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-test.status 2024-09-18T01:07:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-test.status 2024-09-18T01:07:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:07:19.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:07:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-18T01:07:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:07:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:19.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-18T01:07:21.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 29 2024-09-18T01:07:21.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:07:21.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:07:21.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:21.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:21.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:07:21.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-18T01:07:21.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:07:21.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:07:21.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=29 2024-09-18T01:07:21.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:07:21.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:07:21.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:07:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:07:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:07:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:07:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:07:21.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:07:21.658 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:07:21.658 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:07:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:07:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:07:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:07:21.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:07:21.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 29 = '' 2024-09-18T01:07:21.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:07:21.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:07:22.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:07:22.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:07:22.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:22.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:22.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:07:22.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:07:22.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:07:22.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:07:22.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:07:22.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:07:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:07:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:07:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:07:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 29 = '' 2024-09-18T01:07:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:07:22.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:07:22.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:07:22.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:07:22.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:22.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:22.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:07:22.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:07:22.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:07:22.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:07:22.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:07:22.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:07:22.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:07:22.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:07:22.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:07:22.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 29 = '' 2024-09-18T01:07:22.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:07:22.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:07:24.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:07:24.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:07:24.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:24.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:24.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:07:24.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:07:24.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:07:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:07:24.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=29 2024-09-18T01:07:24.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 29 = 29 2024-09-18T01:07:24.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:07:24.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror test snap9 2024-09-18T01:07:24.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:07:24.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:24.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:24.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap9 2024-09-18T01:07:24.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:07:24.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:07:24.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:07:25.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/test@snap9 2024-09-18T01:07:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T01:07:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:07:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:07:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 18 2024-09-18T01:07:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc231b3fe3 2024-09-18T01:07:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc231b3fe3 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:06:29 2024 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:07:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:07:25.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:07:25.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snap8-clean ']' 2024-09-18T01:07:25.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap8-clean 2024-09-18T01:07:25.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:25.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:25.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap8-clean 2024-09-18T01:07:25.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:07:25.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:07:25.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:07:25.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:07:25.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:07:26.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:07:26.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:07:26.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:07:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:07:27.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:07:27.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:07:27.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap9 2024-09-18T01:07:27.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:27.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:27.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap9 2024-09-18T01:07:27.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:07:27.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:07:27.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:07:27.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:07:27.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:07:27.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:07:27.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:07:28.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:07:28.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:07:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:07:28.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:07:28.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ clean_snap_name=snap9-clean 2024-09-18T01:07:28.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap9-clean 2024-09-18T01:07:28.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:07:28.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:28.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:28.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap9-clean 2024-09-18T01:07:28.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20 2024-09-18T01:07:29.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:07:29.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror test 2024-09-18T01:07:29.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:07:29.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:29.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:29.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:07:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=10 2024-09-18T01:07:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:07:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-09-18T01:07:29.490 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 5722112 bytes 2024-09-18T01:07:29.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 30763008 bytes 2024-09-18T01:07:29.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 50364416 bytes 2024-09-18T01:07:29.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 139356160 bytes 2024-09-18T01:07:30.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 202317824 bytes 2024-09-18T01:07:30.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 241734656 bytes 2024-09-18T01:07:30.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 296785920 bytes 2024-09-18T01:07:30.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 304509952 bytes 2024-09-18T01:07:30.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 345131008 bytes 2024-09-18T01:07:30.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 351185920 bytes 2024-09-18T01:07:30.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 358275072 bytes 2024-09-18T01:07:30.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 358896640 bytes 2024-09-18T01:07:30.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 398489600 bytes 2024-09-18T01:07:30.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 399120384 bytes 2024-09-18T01:07:30.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 401862656 bytes 2024-09-18T01:07:31.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 449254400 bytes 2024-09-18T01:07:31.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 509208576 bytes 2024-09-18T01:07:31.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 531225600 bytes 2024-09-18T01:07:31.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 561925120 bytes 2024-09-18T01:07:31.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 575139840 bytes 2024-09-18T01:07:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 643526656 bytes 2024-09-18T01:07:31.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 670820352 bytes 2024-09-18T01:07:31.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 682168320 bytes 2024-09-18T01:07:31.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 777449472 bytes 2024-09-18T01:07:31.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 800829440 bytes 2024-09-18T01:07:32.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 812935168 bytes 2024-09-18T01:07:32.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 861677568 bytes 2024-09-18T01:07:32.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 862321664 bytes 2024-09-18T01:07:32.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 915513344 bytes 2024-09-18T01:07:32.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 943304704 bytes 2024-09-18T01:07:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 986261504 bytes 2024-09-18T01:07:32.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1055919104 bytes 2024-09-18T01:07:32.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1056521216 bytes 2024-09-18T01:07:32.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1076340736 bytes 2024-09-18T01:07:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1106085888 bytes 2024-09-18T01:07:33.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1130174464 bytes 2024-09-18T01:07:33.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1147830272 bytes 2024-09-18T01:07:33.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1179464704 bytes 2024-09-18T01:07:33.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1207772160 bytes 2024-09-18T01:07:33.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1217320960 bytes 2024-09-18T01:07:33.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1264295936 bytes 2024-09-18T01:07:33.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1320217600 bytes 2024-09-18T01:07:33.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1358307328 bytes 2024-09-18T01:07:34.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1432090624 bytes 2024-09-18T01:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1459207168 bytes 2024-09-18T01:07:34.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1511709696 bytes 2024-09-18T01:07:34.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1538247680 bytes 2024-09-18T01:07:34.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1579023360 bytes 2024-09-18T01:07:34.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1579641856 bytes 2024-09-18T01:07:34.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1767270400 bytes 2024-09-18T01:07:34.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 1812115456 bytes 2024-09-18T01:07:35.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 1931514880 bytes 2024-09-18T01:07:35.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 1976801280 bytes 2024-09-18T01:07:35.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2016801792 bytes 2024-09-18T01:07:35.424 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2058706944 bytes 2024-09-18T01:07:35.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2135218176 bytes 2024-09-18T01:07:35.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2140672000 bytes 2024-09-18T01:07:35.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2159817728 bytes 2024-09-18T01:07:35.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2181509120 bytes 2024-09-18T01:07:36.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2260952064 bytes 2024-09-18T01:07:36.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2339950592 bytes 2024-09-18T01:07:36.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2340567040 bytes 2024-09-18T01:07:36.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2366472192 bytes 2024-09-18T01:07:36.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2463413248 bytes 2024-09-18T01:07:36.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2499252224 bytes 2024-09-18T01:07:36.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2582379520 bytes 2024-09-18T01:07:36.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2604291072 bytes 2024-09-18T01:07:37.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2666243072 bytes 2024-09-18T01:07:37.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2696057856 bytes 2024-09-18T01:07:37.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2725673984 bytes 2024-09-18T01:07:37.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2773388288 bytes 2024-09-18T01:07:37.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2854031360 bytes 2024-09-18T01:07:37.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 2882847744 bytes 2024-09-18T01:07:37.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 2916082688 bytes 2024-09-18T01:07:37.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 2950247424 bytes 2024-09-18T01:07:37.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 2986137600 bytes 2024-09-18T01:07:38.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3023844352 bytes 2024-09-18T01:07:38.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3051288576 bytes 2024-09-18T01:07:38.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3090467840 bytes 2024-09-18T01:07:38.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3099617280 bytes 2024-09-18T01:07:38.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3109249024 bytes 2024-09-18T01:07:38.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3129007104 bytes 2024-09-18T01:07:38.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3226177536 bytes 2024-09-18T01:07:38.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3268608000 bytes 2024-09-18T01:07:38.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3316348928 bytes 2024-09-18T01:07:39.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3371829248 bytes 2024-09-18T01:07:39.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3412835328 bytes 2024-09-18T01:07:39.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3455155200 bytes 2024-09-18T01:07:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:07:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:07:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:07:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:07:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-09-18T01:07:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:07:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:39.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:07:39.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:07:39.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:07:39.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:07:40.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-09-18T01:07:40.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:07:40.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:40.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:40.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:40.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local description_pattern= 2024-09-18T01:07:40.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local service_pattern= 2024-09-18T01:07:40.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:07:40.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:07:40.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-09-18T01:07:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='test: 2024-09-18T01:07:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: global_id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:07:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+replaying 2024-09-18T01:07:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: replaying, {"bytes_per_second":17892795.73,"bytes_per_snapshot":536086528.0,"last_snapshot_bytes":536783872,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621640,"remote_snapshot_timestamp":1726621640,"replay_state":"idle"} 2024-09-18T01:07:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: service: mirror.1 on smithi150 2024-09-18T01:07:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:07:33 2024-09-18T01:07:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: peer_sites: 2024-09-18T01:07:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: name: cluster2 2024-09-18T01:07:40.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: state: up+stopped 2024-09-18T01:07:40.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: description: local image is primary 2024-09-18T01:07:40.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: last_update: 2024-09-18 01:07:33' 2024-09-18T01:07:40.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ state: .*up+replaying' 2024-09-18T01:07:40.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: state: up+replaying 2024-09-18T01:07:40.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^ description: .*' 2024-09-18T01:07:40.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: description: replaying, {"bytes_per_second":17892795.73,"bytes_per_snapshot":536086528.0,"last_snapshot_bytes":536783872,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621640,"remote_snapshot_timestamp":1726621640,"replay_state":"idle"} 2024-09-18T01:07:40.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:07:40.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:07:40.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep '^up+' 2024-09-18T01:07:40.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:07:40.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep 'service: *mirror..* on ' 2024-09-18T01:07:40.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: service: mirror.1 on smithi150 2024-09-18T01:07:40.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update 2024-09-18T01:07:40.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-09-18T01:07:40.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:07:33' 2024-09-18T01:07:40.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-09-18 01:07:33' 2024-09-18T01:07:40.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:07:40.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:40.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:40.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state_pattern=up+replaying 2024-09-18T01:07:40.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local 'prev_last_update=2024-09-18 01:07:33' 2024-09-18T01:07:40.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status 2024-09-18T01:07:40.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ CEPH_ARGS= 2024-09-18T01:07:40.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-09-18T01:07:40.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status='OKOKOK142134284mirror.1smithi15019.1.1-330-gc2ddf972trueOK44134416mirror.0smithi15019.1.1-330-gc2ddf972falseOK44814484mirror.2smithi15019.1.1-330-gc2ddf972falseOK45444547mirror.3smithi15019.1.1-330-gc2ddf972falseOKtestc32f6c34-fc35-4674-a05b-f4b6c1db7403up+replayingreplaying, {"bytes_per_second":17892795.73,"bytes_per_snapshot":536086528.0,"last_snapshot_bytes":536783872,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1726621640,"remote_snapshot_timestamp":1726621640,"replay_state":"idle"}42134284mirror.1smithi1502024-09-18 01:07:33cluster2b28075f7-24f5-441c-873b-85d148cb477bup+stoppedlocal image is primary2024-09-18 01:07:33' 2024-09-18T01:07:40.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_update state 2024-09-18T01:07:40.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-09-18T01:07:40.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ last_update='2024-09-18 01:07:33' 2024-09-18T01:07:40.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-09-18T01:07:40.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=up+replaying 2024-09-18T01:07:40.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo up+replaying 2024-09-18T01:07:40.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep up+replaying 2024-09-18T01:07:40.458 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:up+replaying 2024-09-18T01:07:40.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:07:40.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:07:40.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap10 2024-09-18T01:07:40.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap10 2024-09-18T01:07:40.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:07:40.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:07:40.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:07:40.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap10 2024-09-18T01:07:40.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:07:44.317 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8540695277103842) > weight (0.25) 2024-09-18T01:07:44.317 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5084569953265712) > weight (0.25) 2024-09-18T01:07:44.317 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9138651510222103) > weight (0.25) 2024-09-18T01:07:44.317 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.3354650432516474) > weight (0.25) 2024-09-18T01:07:44.317 INFO:tasks.rbd_mirror_thrash:waiting for 108 secs before thrashing 2024-09-18T01:08:10.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T01:08:10.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:10.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:10.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:10.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T01:08:10.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:10.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:10.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:10.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:08:10.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:08:10.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:08:10.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:08:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T01:08:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:11.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:08:11.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:08:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:08:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:08:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:08:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T01:08:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:08:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:08:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:08:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:08:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:08:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:08:11.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:08:11.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:08:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:08:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:08:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:08:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:08:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:08:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:08:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:08:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:08:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:08:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:08:11.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:08:11.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:08:11.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:08:11.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:08:11.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T01:08:12.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T01:08:12.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:08:12.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:08:12.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:08:12.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T01:08:12.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:08:12.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:08:12.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:08:12.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:08:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:08:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:08:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:08:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:08:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:08:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:08:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:08:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:08:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:08:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:08:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:08:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:08:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:08:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:08:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:08:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:08:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:08:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:08:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:08:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:08:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:08:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:08:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:08:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:08:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:08:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:08:12.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:08:12.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:08:12.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:08:12.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T01:08:12.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:08:12.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:08:12.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:08:12.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:08:12.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:08:12.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:08:12.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:08:12.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:12.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-18T01:08:12.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:08:12.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-test.status 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-test.status 2024-09-18T01:08:12.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:08:12.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:08:12.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-test.status 2024-09-18T01:08:12.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-test.status 2024-09-18T01:08:12.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:08:12.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:12.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-18T01:08:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:12.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-18T01:08:13.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 32 2024-09-18T01:08:13.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:08:13.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:13.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:13.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:13.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:08:13.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-18T01:08:13.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:08:14.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:08:14.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=32 2024-09-18T01:08:14.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:08:14.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:08:14.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:08:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:08:14.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:08:14.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:08:14.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:08:14.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:08:14.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:08:14.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:08:14.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:08:14.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 32 = '' 2024-09-18T01:08:14.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:08:14.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:08:14.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:08:14.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:08:14.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:08:14.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:08:14.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:08:14.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:08:14.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:08:14.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:08:14.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 32 = '' 2024-09-18T01:08:14.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:08:14.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:08:15.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:15.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:15.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:15.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:15.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:15.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:08:15.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:08:15.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:08:15.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:08:15.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:08:15.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:08:15.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:08:15.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:08:15.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 32 = '' 2024-09-18T01:08:15.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:08:15.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:08:17.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:17.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:17.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:17.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:17.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:17.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:08:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:08:17.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:08:17.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=32 2024-09-18T01:08:17.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 32 = 32 2024-09-18T01:08:17.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:17.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror test snap10 2024-09-18T01:08:17.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:17.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:17.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:17.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap10 2024-09-18T01:08:17.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:08:17.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:08:17.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:08:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/test@snap10 2024-09-18T01:08:18.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T01:08:18.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:08:18.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:18.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 20 2024-09-18T01:08:18.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc231b3fe3 2024-09-18T01:08:18.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc231b3fe3 2024-09-18T01:08:18.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:08:18.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:08:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:08:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:08:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:08:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:06:29 2024 2024-09-18T01:08:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:08:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:08:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:08:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:08:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:08:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:08:18.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:18.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snap9-clean ']' 2024-09-18T01:08:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap9-clean 2024-09-18T01:08:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap9-clean 2024-09-18T01:08:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:18.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:18.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:18.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:19.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:20.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:20.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:20.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap10 2024-09-18T01:08:20.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:20.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:20.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap10 2024-09-18T01:08:20.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:20.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:20.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:20.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:20.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:20.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:20.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:21.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:21.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:21.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ clean_snap_name=snap10-clean 2024-09-18T01:08:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror test snap10-clean 2024-09-18T01:08:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap10-clean 2024-09-18T01:08:21.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20 2024-09-18T01:08:21.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-09-18T01:08:21.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:21.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:21.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:21.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-09-18T01:08:21.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:21.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:21.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:21.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:08:21.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:08:21.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:08:21.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:08:22.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror test started 2024-09-18T01:08:22.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:22.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:22.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:22.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:08:22.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:08:22.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:08:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-09-18T01:08:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:08:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:08:22.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:08:22.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:08:22.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:08:22.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:08:22.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:08:22.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:08:22.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:08:22.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:08:22.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:08:22.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:08:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:08:22.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:08:22.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:08:22.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:08:22.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:08:22.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:08:22.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:08:22.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:08:22.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:08:22.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:08:22.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-09-18T01:08:22.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-09-18T01:08:22.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:08:22.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:08:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:08:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-09-18T01:08:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:08:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:08:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:08:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:08:22.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:08:22.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:08:22.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:08:22.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:08:22.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:08:22.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:08:22.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:08:22.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:08:22.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:08:22.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:08:22.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:08:22.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:08:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:08:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:08:22.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:08:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:08:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:08:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:08:22.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-09-18T01:08:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:08:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:08:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:08:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:08:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:08:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:08:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:08:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-09-18T01:08:23.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:08:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:08:23.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:08:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-09-18T01:08:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:08:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:23.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-test.status 2024-09-18T01:08:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-test.status 2024-09-18T01:08:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:08:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:08:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-test.status 2024-09-18T01:08:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-test.status 2024-09-18T01:08:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:08:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-09-18T01:08:23.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:23.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-09-18T01:08:24.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 34 2024-09-18T01:08:24.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:08:24.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:24.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:24.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:24.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.status 2024-09-18T01:08:24.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-09-18T01:08:24.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:08:24.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:08:24.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=34 2024-09-18T01:08:24.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:08:24.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:08:24.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:08:25.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:25.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:25.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:25.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:25.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:08:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:08:25.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:08:25.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:08:25.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:08:25.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:08:25.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:08:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:08:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 34 = '' 2024-09-18T01:08:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:08:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:08:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:25.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.status 2024-09-18T01:08:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-09-18T01:08:25.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:08:25.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:08:25.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=34 2024-09-18T01:08:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 34 = 34 2024-09-18T01:08:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean 2024-09-18T01:08:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:08:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap10-clean 2024-09-18T01:08:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:08:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:08:25.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:08:26.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean 2024-09-18T01:08:26.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'test': 2024-09-18T01:08:26.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:08:26.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:08:26.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 21 2024-09-18T01:08:26.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc231b3fe3 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc231b3fe3 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:08:18 2024 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 00:59:36 2024 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c32f6c34-fc35-4674-a05b-f4b6c1db7403 2024-09-18T01:08:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:08:26.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:26.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 10 2024-09-18T01:08:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:26.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap1 2024-09-18T01:08:26.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap1 2024-09-18T01:08:26.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:26.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:26.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap1 2024-09-18T01:08:26.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:26.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:26.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:26.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:26.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:27.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:27.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:27.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:27.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:28.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap1-clean 2024-09-18T01:08:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap1-clean 2024-09-18T01:08:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap1-clean 2024-09-18T01:08:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:28.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:28.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:28.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:29.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:29.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:29.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap2 2024-09-18T01:08:29.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap2 2024-09-18T01:08:29.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:29.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:29.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap2 2024-09-18T01:08:29.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:29.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:29.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:29.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:29.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:30.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:30.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:31.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:31.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:31.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:31.379 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5873896319876728) > weight (0.25) 2024-09-18T01:08:31.379 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9432504935687064) > weight (0.25) 2024-09-18T01:08:31.379 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6388872927978012) > weight (0.25) 2024-09-18T01:08:31.379 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6160668749969955) > weight (0.25) 2024-09-18T01:08:31.379 INFO:tasks.rbd_mirror_thrash:waiting for 114 secs before thrashing 2024-09-18T01:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap2-clean 2024-09-18T01:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap2-clean 2024-09-18T01:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap2-clean 2024-09-18T01:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:31.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:31.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:31.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:32.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:32.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:32.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:33.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap3 2024-09-18T01:08:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap3 2024-09-18T01:08:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap3 2024-09-18T01:08:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:33.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:33.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:33.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:33.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:34.338 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:34.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap3-clean 2024-09-18T01:08:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap3-clean 2024-09-18T01:08:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap3-clean 2024-09-18T01:08:34.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:34.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:34.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:34.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:34.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:35.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:35.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:35.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:35.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:36.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:36.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:36.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:36.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap4 2024-09-18T01:08:36.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap4 2024-09-18T01:08:36.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:36.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:36.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap4 2024-09-18T01:08:36.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:36.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:36.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:36.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:36.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:36.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:36.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:37.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:37.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:37.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap4-clean 2024-09-18T01:08:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap4-clean 2024-09-18T01:08:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap4-clean 2024-09-18T01:08:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:37.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:38.391 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:38.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:39.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:39.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap5 2024-09-18T01:08:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap5 2024-09-18T01:08:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap5 2024-09-18T01:08:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:39.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:39.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:39.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:39.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:39.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:40.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:40.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:41.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:41.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap5-clean 2024-09-18T01:08:41.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap5-clean 2024-09-18T01:08:41.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:41.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:41.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap5-clean 2024-09-18T01:08:41.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:41.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:41.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:41.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:41.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:41.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:41.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:42.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:42.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:42.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:42.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:42.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:42.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap6 2024-09-18T01:08:42.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap6 2024-09-18T01:08:42.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:42.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:42.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap6 2024-09-18T01:08:42.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:42.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:42.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:42.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:42.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:43.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:43.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:43.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:43.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:44.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap6-clean 2024-09-18T01:08:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap6-clean 2024-09-18T01:08:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap6-clean 2024-09-18T01:08:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:44.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:44.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:45.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:45.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:45.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:45.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap7 2024-09-18T01:08:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap7 2024-09-18T01:08:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap7 2024-09-18T01:08:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:45.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:45.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:46.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:46.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:47.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:47.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:47.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:47.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:47.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap7-clean 2024-09-18T01:08:47.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap7-clean 2024-09-18T01:08:47.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:47.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:47.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap7-clean 2024-09-18T01:08:47.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:47.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:47.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:47.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:47.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:47.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:47.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:48.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:48.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:48.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:49.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:49.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap8 2024-09-18T01:08:49.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap8 2024-09-18T01:08:49.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:49.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:49.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap8 2024-09-18T01:08:49.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:49.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:49.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:49.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:49.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:49.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:49.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:50.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:50.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:50.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap8-clean 2024-09-18T01:08:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap8-clean 2024-09-18T01:08:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap8-clean 2024-09-18T01:08:50.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:50.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:50.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:51.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:51.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:51.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:51.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:52.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:52.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:52.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap9 2024-09-18T01:08:52.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap9 2024-09-18T01:08:52.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:52.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:52.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap9 2024-09-18T01:08:52.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:52.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:52.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:52.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:52.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:52.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:52.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:53.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:53.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:53.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:53.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:53.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap9-clean 2024-09-18T01:08:53.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap9-clean 2024-09-18T01:08:53.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:53.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:53.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap9-clean 2024-09-18T01:08:53.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:53.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:53.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:53.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:53.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:54.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:54.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:54.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:54.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:55.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:55.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:55.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap10 2024-09-18T01:08:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap10 2024-09-18T01:08:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap10 2024-09-18T01:08:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:55.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:55.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:55.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:56.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:56.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap10-clean 2024-09-18T01:08:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror test snap10-clean 2024-09-18T01:08:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap10-clean 2024-09-18T01:08:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:08:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:56.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:56.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:56.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export 2024-09-18T01:08:57.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:57.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:58.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-09-18T01:08:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:58.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-test.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-test.export 2024-09-18T01:08:58.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:08:58.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 10 2024-09-18T01:08:58.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:58.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap1 2024-09-18T01:08:58.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap1 2024-09-18T01:08:58.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:58.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:58.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:58.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap1 2024-09-18T01:08:58.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1 2024-09-18T01:08:58.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:08:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap1-clean 2024-09-18T01:08:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap1-clean 2024-09-18T01:08:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap1-clean 2024-09-18T01:08:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean 2024-09-18T01:08:59.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:08:59.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:08:59.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap2 2024-09-18T01:08:59.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap2 2024-09-18T01:08:59.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:08:59.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:08:59.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:08:59.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap2 2024-09-18T01:08:59.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2 2024-09-18T01:09:00.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:00.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap2-clean 2024-09-18T01:09:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap2-clean 2024-09-18T01:09:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap2-clean 2024-09-18T01:09:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean 2024-09-18T01:09:01.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:09:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap3 2024-09-18T01:09:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap3 2024-09-18T01:09:01.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap3 2024-09-18T01:09:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3 2024-09-18T01:09:02.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:02.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap3-clean 2024-09-18T01:09:02.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap3-clean 2024-09-18T01:09:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap3-clean 2024-09-18T01:09:02.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean 2024-09-18T01:09:03.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:09:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap4 2024-09-18T01:09:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap4 2024-09-18T01:09:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap4 2024-09-18T01:09:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4 2024-09-18T01:09:04.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:04.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap4-clean 2024-09-18T01:09:04.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap4-clean 2024-09-18T01:09:04.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:04.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:04.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:04.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap4-clean 2024-09-18T01:09:04.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean 2024-09-18T01:09:05.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:09:05.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap5 2024-09-18T01:09:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap5 2024-09-18T01:09:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap5 2024-09-18T01:09:05.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5 2024-09-18T01:09:06.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap5-clean 2024-09-18T01:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap5-clean 2024-09-18T01:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap5-clean 2024-09-18T01:09:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean 2024-09-18T01:09:07.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:07.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:09:07.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap6 2024-09-18T01:09:07.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap6 2024-09-18T01:09:07.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:07.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:07.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:07.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap6 2024-09-18T01:09:07.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6 2024-09-18T01:09:08.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap6-clean 2024-09-18T01:09:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap6-clean 2024-09-18T01:09:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap6-clean 2024-09-18T01:09:08.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean 2024-09-18T01:09:09.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:09:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap7 2024-09-18T01:09:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap7 2024-09-18T01:09:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap7 2024-09-18T01:09:09.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7 2024-09-18T01:09:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap7-clean 2024-09-18T01:09:10.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap7-clean 2024-09-18T01:09:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap7-clean 2024-09-18T01:09:10.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean 2024-09-18T01:09:11.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:09:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap8 2024-09-18T01:09:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap8 2024-09-18T01:09:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap8 2024-09-18T01:09:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8 2024-09-18T01:09:12.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap8-clean 2024-09-18T01:09:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap8-clean 2024-09-18T01:09:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap8-clean 2024-09-18T01:09:12.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean 2024-09-18T01:09:13.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap9 2024-09-18T01:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap9 2024-09-18T01:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap9 2024-09-18T01:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9 2024-09-18T01:09:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:14.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap9-clean 2024-09-18T01:09:14.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap9-clean 2024-09-18T01:09:14.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:14.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:14.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:14.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap9-clean 2024-09-18T01:09:14.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean 2024-09-18T01:09:15.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:15.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:09:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap10 2024-09-18T01:09:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap10 2024-09-18T01:09:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap10 2024-09-18T01:09:15.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10 2024-09-18T01:09:16.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:16.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap10-clean 2024-09-18T01:09:16.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror test snap10-clean 2024-09-18T01:09:16.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:16.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:16.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:16.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap10-clean 2024-09-18T01:09:16.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean 2024-09-18T01:09:17.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:09:17.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror test 2024-09-18T01:09:17.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:17.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:17.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:17.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:09:17.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:09:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror test 2024-09-18T01:09:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:17.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2024-09-18T01:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/test 2024-09-18T01:09:18.144 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:09:18.142+0000 7fef680a7640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x556e807aab60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:09:18.144 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:09:18.142+0000 7fef6e0b3640 -1 rbd::mirror::ImageReplayer: 0x556e80f4a500 [3/c32f6c34-fc35-4674-a05b-f4b6c1db7403] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:09:19.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror test deleted 2024-09-18T01:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:09:19.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror test 2024-09-18T01:09:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:09:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:09:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=test 2024-09-18T01:09:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/test 2024-09-18T01:09:19.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:09:19.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image test: (2) No such file or directory 2024-09-18T01:09:19.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:09:19.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:09:19.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:09:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror test deleted '' 2024-09-18T01:09:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:09:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=test 2024-09-18T01:09:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:09:19.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:09:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:09:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:09:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror test 2024-09-18T01:09:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:09:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:09:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=test 2024-09-18T01:09:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/test 2024-09-18T01:09:19.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:09:19.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image test: (2) No such file or directory 2024-09-18T01:09:19.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:09:19.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:09:19.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:09:19.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:09:19.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ testlog 'TEST: create many images' 2024-09-18T01:09:19.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ tee -a /tmp/tmp.yeNdCEAq6i/rbd-mirror.test.log 2024-09-18T01:09:19.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ date '+%F %T' 2024-09-18T01:09:19.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo 2024-09-18 01:09:19 TEST: create many images 2024-09-18T01:09:19.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:2024-09-18 01:09:19 TEST: create many images 2024-09-18T01:09:19.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ snap_name=snap 2024-09-18T01:09:19.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 50 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_1 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 snapshot 128M 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:09:19.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_1 128M 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:19.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_1 2024-09-18T01:09:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:09:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:09:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:09:19.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_1 snapshot 2024-09-18T01:09:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:09:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:09:19.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2024-09-18T01:09:19.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:09:19.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_1': 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 122961ba68e1 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.122961ba68e1 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:09:19 2024 2024-09-18T01:09:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:09:19 2024 2024-09-18T01:09:20.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:09:19 2024 2024-09-18T01:09:20.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:09:20.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:09:20.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 1bfda626-752e-40be-8f8c-5733d4c53ac8 2024-09-18T01:09:20.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:09:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:09:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_1 2024-09-18T01:09:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:09:20.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:09:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=7 2024-09-18T01:09:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:09:20.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:09:20.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 30597120 bytes 2024-09-18T01:09:20.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 83003392 bytes 2024-09-18T01:09:20.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 132635648 bytes 2024-09-18T01:09:20.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 206150656 bytes 2024-09-18T01:09:20.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 244970496 bytes 2024-09-18T01:09:20.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 258771968 bytes 2024-09-18T01:09:20.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 331568128 bytes 2024-09-18T01:09:20.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 372922368 bytes 2024-09-18T01:09:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 420306944 bytes 2024-09-18T01:09:21.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 450743296 bytes 2024-09-18T01:09:21.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 515966976 bytes 2024-09-18T01:09:21.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 606830592 bytes 2024-09-18T01:09:21.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 626071552 bytes 2024-09-18T01:09:21.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 656278528 bytes 2024-09-18T01:09:21.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 683495424 bytes 2024-09-18T01:09:21.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 707343360 bytes 2024-09-18T01:09:21.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 734206976 bytes 2024-09-18T01:09:21.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 750205952 bytes 2024-09-18T01:09:22.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 775207936 bytes 2024-09-18T01:09:22.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 811595776 bytes 2024-09-18T01:09:22.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 833464320 bytes 2024-09-18T01:09:22.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 908372992 bytes 2024-09-18T01:09:22.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 936373248 bytes 2024-09-18T01:09:22.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 979613696 bytes 2024-09-18T01:09:22.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1015956480 bytes 2024-09-18T01:09:22.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1076652032 bytes 2024-09-18T01:09:22.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1090769920 bytes 2024-09-18T01:09:22.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1123531776 bytes 2024-09-18T01:09:23.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1210016768 bytes 2024-09-18T01:09:23.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1297408000 bytes 2024-09-18T01:09:23.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1305407488 bytes 2024-09-18T01:09:23.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1347720192 bytes 2024-09-18T01:09:23.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1427555328 bytes 2024-09-18T01:09:23.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1499220992 bytes 2024-09-18T01:09:23.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1588368384 bytes 2024-09-18T01:09:23.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1637775360 bytes 2024-09-18T01:09:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1640284160 bytes 2024-09-18T01:09:24.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1705742336 bytes 2024-09-18T01:09:24.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1792167936 bytes 2024-09-18T01:09:24.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1811070976 bytes 2024-09-18T01:09:24.423 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1836195840 bytes 2024-09-18T01:09:24.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1851954176 bytes 2024-09-18T01:09:24.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1875903488 bytes 2024-09-18T01:09:24.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1921809408 bytes 2024-09-18T01:09:24.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1955166208 bytes 2024-09-18T01:09:24.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2039287808 bytes 2024-09-18T01:09:25.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2093441024 bytes 2024-09-18T01:09:25.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2156773376 bytes 2024-09-18T01:09:25.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2189526016 bytes 2024-09-18T01:09:25.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2195372032 bytes 2024-09-18T01:09:25.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2249164800 bytes 2024-09-18T01:09:25.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2322663424 bytes 2024-09-18T01:09:25.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2352552960 bytes 2024-09-18T01:09:25.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2374723584 bytes 2024-09-18T01:09:25.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2375357440 bytes 2024-09-18T01:09:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2403980288 bytes 2024-09-18T01:09:26.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2419437568 bytes 2024-09-18T01:09:26.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2473558016 bytes 2024-09-18T01:09:26.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2564637696 bytes 2024-09-18T01:09:26.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2585712640 bytes 2024-09-18T01:09:26.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2645239808 bytes 2024-09-18T01:09:26.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2673501184 bytes 2024-09-18T01:09:26.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2747553792 bytes 2024-09-18T01:09:26.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2829095936 bytes 2024-09-18T01:09:26.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2869357568 bytes 2024-09-18T01:09:27.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:09:27.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:09:27.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_2 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 snapshot 128M 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:09:27.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_2 128M 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:27.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_2 2024-09-18T01:09:27.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:09:27.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:27.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:09:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:09:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_2 snapshot 2024-09-18T01:09:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:09:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:09:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2024-09-18T01:09:27.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:09:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2024-09-18T01:09:27.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_2': 2024-09-18T01:09:27.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:09:27.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:09:27.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:09:27.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1231dcea0cd2 2024-09-18T01:09:27.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1231dcea0cd2 2024-09-18T01:09:27.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:09:27.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:09:27.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:09:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:09:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:09:27 2024 2024-09-18T01:09:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:09:27 2024 2024-09-18T01:09:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:09:27 2024 2024-09-18T01:09:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:09:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:09:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 0efe7b51-8742-400d-adca-bf1d51dea26b 2024-09-18T01:09:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:09:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:09:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_2 2024-09-18T01:09:27.908 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:27.908 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:27.908 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:09:27.908 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:09:27.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:09:27.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:09:27.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:09:28.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 11334656 bytes 2024-09-18T01:09:28.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 85899264 bytes 2024-09-18T01:09:28.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 130290688 bytes 2024-09-18T01:09:28.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 161580032 bytes 2024-09-18T01:09:28.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 170014720 bytes 2024-09-18T01:09:28.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 189148160 bytes 2024-09-18T01:09:28.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 229783552 bytes 2024-09-18T01:09:28.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 266748928 bytes 2024-09-18T01:09:28.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 319545344 bytes 2024-09-18T01:09:28.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 339781632 bytes 2024-09-18T01:09:29.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 371737600 bytes 2024-09-18T01:09:29.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 380846080 bytes 2024-09-18T01:09:29.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 383876096 bytes 2024-09-18T01:09:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 416534528 bytes 2024-09-18T01:09:29.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 448069632 bytes 2024-09-18T01:09:29.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 474372096 bytes 2024-09-18T01:09:29.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 512238592 bytes 2024-09-18T01:09:29.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 558484480 bytes 2024-09-18T01:09:29.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 602836992 bytes 2024-09-18T01:09:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 650417152 bytes 2024-09-18T01:09:30.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 659181568 bytes 2024-09-18T01:09:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 675336192 bytes 2024-09-18T01:09:30.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 722294784 bytes 2024-09-18T01:09:30.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 777857024 bytes 2024-09-18T01:09:30.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 812120064 bytes 2024-09-18T01:09:30.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 840285184 bytes 2024-09-18T01:09:30.546 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:09:30.544+0000 7f80d5d16640 -1 reset not still connected to 0x56418c645c70 2024-09-18T01:09:30.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 851790848 bytes 2024-09-18T01:09:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 868052992 bytes 2024-09-18T01:09:30.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 952695808 bytes 2024-09-18T01:09:30.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 971222016 bytes 2024-09-18T01:09:31.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1013068800 bytes 2024-09-18T01:09:31.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1063689216 bytes 2024-09-18T01:09:31.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1083381760 bytes 2024-09-18T01:09:31.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1103818752 bytes 2024-09-18T01:09:31.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1126574080 bytes 2024-09-18T01:09:31.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1169073152 bytes 2024-09-18T01:09:31.658 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1235920896 bytes 2024-09-18T01:09:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1282864128 bytes 2024-09-18T01:09:31.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1361700864 bytes 2024-09-18T01:09:32.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1445585920 bytes 2024-09-18T01:09:32.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1451726848 bytes 2024-09-18T01:09:32.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1491890176 bytes 2024-09-18T01:09:32.318 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6290990123338305) > weight (0.25) 2024-09-18T01:09:32.319 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6118977278126677) > weight (0.25) 2024-09-18T01:09:32.319 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-09-18T01:09:32.319 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-09-18T01:09:32.319 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-18T01:09:32.319 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-09-18T01:09:32.320 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:09:32.321 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:09:32.318+0000 7fbd76f0f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 41314) UID: 1000 2024-09-18T01:09:32.406 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1579524096 bytes 2024-09-18T01:09:32.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1637760000 bytes 2024-09-18T01:09:32.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1654328320 bytes 2024-09-18T01:09:32.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1687355392 bytes 2024-09-18T01:09:32.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1695395840 bytes 2024-09-18T01:09:32.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1741730816 bytes 2024-09-18T01:09:33.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1777518592 bytes 2024-09-18T01:09:33.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1840831488 bytes 2024-09-18T01:09:33.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 1881498624 bytes 2024-09-18T01:09:33.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 1936772096 bytes 2024-09-18T01:09:33.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 1979202560 bytes 2024-09-18T01:09:33.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 1995873280 bytes 2024-09-18T01:09:33.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2048080896 bytes 2024-09-18T01:09:33.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2065466368 bytes 2024-09-18T01:09:33.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2108350464 bytes 2024-09-18T01:09:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2137085952 bytes 2024-09-18T01:09:34.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2137681920 bytes 2024-09-18T01:09:34.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2160845824 bytes 2024-09-18T01:09:34.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2199066624 bytes 2024-09-18T01:09:34.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2231661568 bytes 2024-09-18T01:09:34.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2251209728 bytes 2024-09-18T01:09:34.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2339793920 bytes 2024-09-18T01:09:34.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2399034368 bytes 2024-09-18T01:09:34.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2452987904 bytes 2024-09-18T01:09:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2526529536 bytes 2024-09-18T01:09:35.055 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2557279232 bytes 2024-09-18T01:09:35.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2595761152 bytes 2024-09-18T01:09:35.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2628160512 bytes 2024-09-18T01:09:35.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2663147520 bytes 2024-09-18T01:09:35.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2703920128 bytes 2024-09-18T01:09:35.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 2763740160 bytes 2024-09-18T01:09:35.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 2779422720 bytes 2024-09-18T01:09:35.796 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 2834667520 bytes 2024-09-18T01:09:35.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 2902875136 bytes 2024-09-18T01:09:36.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 2960790528 bytes 2024-09-18T01:09:36.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3002081280 bytes 2024-09-18T01:09:36.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3047152640 bytes 2024-09-18T01:09:36.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3080372224 bytes 2024-09-18T01:09:36.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3128469504 bytes 2024-09-18T01:09:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3238970368 bytes 2024-09-18T01:09:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3294355456 bytes 2024-09-18T01:09:36.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3321891840 bytes 2024-09-18T01:09:36.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3360923648 bytes 2024-09-18T01:09:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3419756544 bytes 2024-09-18T01:09:37.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3420375040 bytes 2024-09-18T01:09:37.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3519203328 bytes 2024-09-18T01:09:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3568033792 bytes 2024-09-18T01:09:37.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3633307648 bytes 2024-09-18T01:09:37.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3645125632 bytes 2024-09-18T01:09:37.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3689767936 bytes 2024-09-18T01:09:37.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3765879808 bytes 2024-09-18T01:09:38.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3766510592 bytes 2024-09-18T01:09:38.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 3848841216 bytes 2024-09-18T01:09:38.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 3885396992 bytes 2024-09-18T01:09:38.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 3920497664 bytes 2024-09-18T01:09:38.421 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-09-18T01:09:38.421 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-18T01:09:38.421 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-18T01:09:38.424 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-18T01:09:38.424 INFO:tasks.rbd_mirror_thrash:waiting for 64 secs before thrashing 2024-09-18T01:09:38.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 3966218240 bytes 2024-09-18T01:09:38.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4005859328 bytes 2024-09-18T01:09:38.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4038260736 bytes 2024-09-18T01:09:38.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4057433088 bytes 2024-09-18T01:09:39.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4109218816 bytes 2024-09-18T01:09:39.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4202619904 bytes 2024-09-18T01:09:39.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4220175360 bytes 2024-09-18T01:09:39.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4261637120 bytes 2024-09-18T01:09:39.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4310878208 bytes 2024-09-18T01:09:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4352499712 bytes 2024-09-18T01:09:39.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4442104832 bytes 2024-09-18T01:09:39.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4507030528 bytes 2024-09-18T01:09:40.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 4551692288 bytes 2024-09-18T01:09:40.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11100 IOs, 4574005248 bytes 2024-09-18T01:09:40.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11200 IOs, 4610640896 bytes 2024-09-18T01:09:40.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11300 IOs, 4696825856 bytes 2024-09-18T01:09:40.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11400 IOs, 4735736832 bytes 2024-09-18T01:09:40.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11500 IOs, 4783386624 bytes 2024-09-18T01:09:40.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11600 IOs, 4824416256 bytes 2024-09-18T01:09:40.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11700 IOs, 4851806208 bytes 2024-09-18T01:09:40.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11800 IOs, 4899553280 bytes 2024-09-18T01:09:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11900 IOs, 4916580352 bytes 2024-09-18T01:09:41.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12000 IOs, 4921991168 bytes 2024-09-18T01:09:41.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12100 IOs, 4976893952 bytes 2024-09-18T01:09:41.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12200 IOs, 4994565120 bytes 2024-09-18T01:09:41.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12300 IOs, 5060595712 bytes 2024-09-18T01:09:41.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12400 IOs, 5061780480 bytes 2024-09-18T01:09:41.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12500 IOs, 5140321280 bytes 2024-09-18T01:09:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:12600 IOs, 5181083648 bytes 2024-09-18T01:09:41.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:09:41.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:09:41.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_3 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 snapshot 128M 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_3 128M 2024-09-18T01:09:41.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:09:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:09:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:09:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:09:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_3 2024-09-18T01:09:41.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:09:41.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:41.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:09:41.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:09:41.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_3 snapshot 2024-09-18T01:09:41.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:41.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:41.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:09:41.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:09:41.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2024-09-18T01:09:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:09:42.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2024-09-18T01:09:42.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_3': 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 123fe8cb54cb 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.123fe8cb54cb 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:09:41 2024 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:09:41 2024 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:09:41 2024 2024-09-18T01:09:42.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:09:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:09:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 26ca4ff3-0678-4f67-8c75-7af181831497 2024-09-18T01:09:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:09:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:09:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_3 2024-09-18T01:09:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:09:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:09:42.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=12 2024-09-18T01:09:42.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:09:42.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:09:43.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 23979008 bytes 2024-09-18T01:09:43.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 93970432 bytes 2024-09-18T01:09:43.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 111883264 bytes 2024-09-18T01:09:43.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 150443008 bytes 2024-09-18T01:09:43.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 151049216 bytes 2024-09-18T01:09:43.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 151660544 bytes 2024-09-18T01:09:43.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 229508096 bytes 2024-09-18T01:09:43.796 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 280729600 bytes 2024-09-18T01:09:43.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 342616064 bytes 2024-09-18T01:09:44.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 391947264 bytes 2024-09-18T01:09:44.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 447172608 bytes 2024-09-18T01:09:44.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 539516928 bytes 2024-09-18T01:09:44.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 588896256 bytes 2024-09-18T01:09:44.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 621919232 bytes 2024-09-18T01:09:44.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 694265856 bytes 2024-09-18T01:09:44.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 775385088 bytes 2024-09-18T01:09:44.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 856292352 bytes 2024-09-18T01:09:45.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 908134400 bytes 2024-09-18T01:09:45.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 925478912 bytes 2024-09-18T01:09:45.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 952345600 bytes 2024-09-18T01:09:45.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 984516608 bytes 2024-09-18T01:09:45.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1028562944 bytes 2024-09-18T01:09:45.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1040309248 bytes 2024-09-18T01:09:45.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1063385088 bytes 2024-09-18T01:09:45.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1085108224 bytes 2024-09-18T01:09:45.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1126671360 bytes 2024-09-18T01:09:45.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1156278272 bytes 2024-09-18T01:09:46.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1192583168 bytes 2024-09-18T01:09:46.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1195987968 bytes 2024-09-18T01:09:46.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1233619968 bytes 2024-09-18T01:09:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1290058752 bytes 2024-09-18T01:09:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1354504192 bytes 2024-09-18T01:09:46.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1451621376 bytes 2024-09-18T01:09:46.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1483131904 bytes 2024-09-18T01:09:46.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1523501056 bytes 2024-09-18T01:09:46.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1588848640 bytes 2024-09-18T01:09:47.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1628660736 bytes 2024-09-18T01:09:47.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1653706752 bytes 2024-09-18T01:09:47.443 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1660572672 bytes 2024-09-18T01:09:47.548 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1694900224 bytes 2024-09-18T01:09:47.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1736404992 bytes 2024-09-18T01:09:47.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1759536128 bytes 2024-09-18T01:09:47.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1760941056 bytes 2024-09-18T01:09:47.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1839953920 bytes 2024-09-18T01:09:48.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1901290496 bytes 2024-09-18T01:09:48.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1990963200 bytes 2024-09-18T01:09:48.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2020167680 bytes 2024-09-18T01:09:48.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2081393664 bytes 2024-09-18T01:09:48.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2143419392 bytes 2024-09-18T01:09:48.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2155825152 bytes 2024-09-18T01:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2231951360 bytes 2024-09-18T01:09:48.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2298206208 bytes 2024-09-18T01:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2408276992 bytes 2024-09-18T01:09:49.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2437213184 bytes 2024-09-18T01:09:49.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2488887296 bytes 2024-09-18T01:09:49.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2566121472 bytes 2024-09-18T01:09:49.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2577064960 bytes 2024-09-18T01:09:49.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2627351552 bytes 2024-09-18T01:09:49.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2658786304 bytes 2024-09-18T01:09:49.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2662560768 bytes 2024-09-18T01:09:49.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2694365184 bytes 2024-09-18T01:09:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2836527104 bytes 2024-09-18T01:09:50.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2837865472 bytes 2024-09-18T01:09:50.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2852322304 bytes 2024-09-18T01:09:50.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2852912128 bytes 2024-09-18T01:09:50.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2875473920 bytes 2024-09-18T01:09:50.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2921368576 bytes 2024-09-18T01:09:50.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2990546944 bytes 2024-09-18T01:09:50.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2999201792 bytes 2024-09-18T01:09:50.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3052919808 bytes 2024-09-18T01:09:51.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3096168448 bytes 2024-09-18T01:09:51.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3141572608 bytes 2024-09-18T01:09:51.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3189098496 bytes 2024-09-18T01:09:51.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3192572928 bytes 2024-09-18T01:09:51.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3280745472 bytes 2024-09-18T01:09:51.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3375055872 bytes 2024-09-18T01:09:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3407683584 bytes 2024-09-18T01:09:51.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3414654976 bytes 2024-09-18T01:09:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3449421824 bytes 2024-09-18T01:09:52.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3492201472 bytes 2024-09-18T01:09:52.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3515381760 bytes 2024-09-18T01:09:52.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3526791168 bytes 2024-09-18T01:09:52.391 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3565475840 bytes 2024-09-18T01:09:52.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3624744960 bytes 2024-09-18T01:09:52.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3660755968 bytes 2024-09-18T01:09:52.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3715850240 bytes 2024-09-18T01:09:52.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3752955904 bytes 2024-09-18T01:09:52.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3754388480 bytes 2024-09-18T01:09:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3773286400 bytes 2024-09-18T01:09:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3818736640 bytes 2024-09-18T01:09:53.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3846267904 bytes 2024-09-18T01:09:53.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3860748288 bytes 2024-09-18T01:09:53.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3885394944 bytes 2024-09-18T01:09:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3913498624 bytes 2024-09-18T01:09:53.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 3948230656 bytes 2024-09-18T01:09:53.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4007698432 bytes 2024-09-18T01:09:53.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4028344320 bytes 2024-09-18T01:09:53.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4048295936 bytes 2024-09-18T01:09:53.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4065905664 bytes 2024-09-18T01:09:54.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4092965888 bytes 2024-09-18T01:09:54.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4141196288 bytes 2024-09-18T01:09:54.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4217408512 bytes 2024-09-18T01:09:54.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4271594496 bytes 2024-09-18T01:09:54.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4300397568 bytes 2024-09-18T01:09:54.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4460824576 bytes 2024-09-18T01:09:54.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4498946048 bytes 2024-09-18T01:09:54.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_4 2024-09-18T01:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 snapshot 128M 2024-09-18T01:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_4 128M 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_4 2024-09-18T01:09:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:09:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:09:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:09:54.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_4 snapshot 2024-09-18T01:09:54.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:54.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:54.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:09:54.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:09:54.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2024-09-18T01:09:55.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:09:55.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2024-09-18T01:09:55.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_4': 2024-09-18T01:09:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:09:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:09:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:09:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1248b1c8a266 2024-09-18T01:09:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1248b1c8a266 2024-09-18T01:09:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:09:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:09:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:09:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:09:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:09:54 2024 2024-09-18T01:09:55.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:09:54 2024 2024-09-18T01:09:55.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:09:54 2024 2024-09-18T01:09:55.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:09:55.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:09:55.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: f06096fd-c27d-4c66-a888-be876f867f5a 2024-09-18T01:09:55.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:09:55.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:09:55.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_4 2024-09-18T01:09:55.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:09:55.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:09:55.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:09:55.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:09:55.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=13 2024-09-18T01:09:55.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:09:55.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:09:55.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 68415488 bytes 2024-09-18T01:09:55.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 127495168 bytes 2024-09-18T01:09:55.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 146345984 bytes 2024-09-18T01:09:55.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 213809152 bytes 2024-09-18T01:09:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 236388352 bytes 2024-09-18T01:09:56.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 275563520 bytes 2024-09-18T01:09:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 297155584 bytes 2024-09-18T01:09:56.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 319838208 bytes 2024-09-18T01:09:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 327474176 bytes 2024-09-18T01:09:56.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 373511168 bytes 2024-09-18T01:09:56.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 446618624 bytes 2024-09-18T01:09:56.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 543390720 bytes 2024-09-18T01:09:56.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 584950784 bytes 2024-09-18T01:09:57.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 621199360 bytes 2024-09-18T01:09:57.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 688201728 bytes 2024-09-18T01:09:57.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 711298048 bytes 2024-09-18T01:09:57.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 714425344 bytes 2024-09-18T01:09:57.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 747727872 bytes 2024-09-18T01:09:57.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 760933376 bytes 2024-09-18T01:09:57.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 783746048 bytes 2024-09-18T01:09:57.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 815164416 bytes 2024-09-18T01:09:57.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 864986112 bytes 2024-09-18T01:09:57.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 903876608 bytes 2024-09-18T01:09:58.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 946721792 bytes 2024-09-18T01:09:58.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 985498624 bytes 2024-09-18T01:09:58.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 991683584 bytes 2024-09-18T01:09:58.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1017647104 bytes 2024-09-18T01:09:58.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1200224256 bytes 2024-09-18T01:09:58.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1217287168 bytes 2024-09-18T01:09:58.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1249067008 bytes 2024-09-18T01:09:59.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1249675264 bytes 2024-09-18T01:09:59.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1283178496 bytes 2024-09-18T01:09:59.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1309779968 bytes 2024-09-18T01:09:59.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1333156864 bytes 2024-09-18T01:09:59.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1352049664 bytes 2024-09-18T01:09:59.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1355832320 bytes 2024-09-18T01:09:59.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1374343168 bytes 2024-09-18T01:09:59.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1387159552 bytes 2024-09-18T01:09:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1405390848 bytes 2024-09-18T01:09:59.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1460478976 bytes 2024-09-18T01:10:00.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1491084288 bytes 2024-09-18T01:10:00.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1531013120 bytes 2024-09-18T01:10:00.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1616671744 bytes 2024-09-18T01:10:00.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1632718848 bytes 2024-09-18T01:10:00.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1676485632 bytes 2024-09-18T01:10:00.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1708559360 bytes 2024-09-18T01:10:00.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1725005824 bytes 2024-09-18T01:10:00.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1749259264 bytes 2024-09-18T01:10:00.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1773612032 bytes 2024-09-18T01:10:00.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1786307584 bytes 2024-09-18T01:10:01.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 1798691840 bytes 2024-09-18T01:10:01.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 1822358528 bytes 2024-09-18T01:10:01.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 1850123264 bytes 2024-09-18T01:10:01.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 1969135616 bytes 2024-09-18T01:10:01.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2000528384 bytes 2024-09-18T01:10:01.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2006870016 bytes 2024-09-18T01:10:01.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2058318848 bytes 2024-09-18T01:10:01.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2093569024 bytes 2024-09-18T01:10:01.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2127899648 bytes 2024-09-18T01:10:02.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2155494400 bytes 2024-09-18T01:10:02.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2208824320 bytes 2024-09-18T01:10:02.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2239822848 bytes 2024-09-18T01:10:02.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2277274624 bytes 2024-09-18T01:10:02.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2310436864 bytes 2024-09-18T01:10:02.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2401376256 bytes 2024-09-18T01:10:02.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2457413632 bytes 2024-09-18T01:10:02.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2508007424 bytes 2024-09-18T01:10:03.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2616795136 bytes 2024-09-18T01:10:03.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2645994496 bytes 2024-09-18T01:10:03.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2659244032 bytes 2024-09-18T01:10:03.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2785050624 bytes 2024-09-18T01:10:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2812496896 bytes 2024-09-18T01:10:03.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 2874929152 bytes 2024-09-18T01:10:03.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 2959106048 bytes 2024-09-18T01:10:03.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 2965420032 bytes 2024-09-18T01:10:04.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3010098176 bytes 2024-09-18T01:10:04.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3023494144 bytes 2024-09-18T01:10:04.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3077370880 bytes 2024-09-18T01:10:04.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3107194880 bytes 2024-09-18T01:10:04.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3162808320 bytes 2024-09-18T01:10:04.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3264237568 bytes 2024-09-18T01:10:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3346593792 bytes 2024-09-18T01:10:04.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3393254400 bytes 2024-09-18T01:10:05.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3445705728 bytes 2024-09-18T01:10:05.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3446967296 bytes 2024-09-18T01:10:05.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3494035456 bytes 2024-09-18T01:10:05.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3561833472 bytes 2024-09-18T01:10:05.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3605828608 bytes 2024-09-18T01:10:05.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3653170176 bytes 2024-09-18T01:10:05.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3711923200 bytes 2024-09-18T01:10:05.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3741701120 bytes 2024-09-18T01:10:06.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3811225600 bytes 2024-09-18T01:10:06.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3860071424 bytes 2024-09-18T01:10:06.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3927165952 bytes 2024-09-18T01:10:06.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 3974123520 bytes 2024-09-18T01:10:06.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4010762240 bytes 2024-09-18T01:10:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4049193984 bytes 2024-09-18T01:10:06.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4089417728 bytes 2024-09-18T01:10:06.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4173260800 bytes 2024-09-18T01:10:07.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4236619776 bytes 2024-09-18T01:10:07.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4297582592 bytes 2024-09-18T01:10:07.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4321246208 bytes 2024-09-18T01:10:07.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4337596416 bytes 2024-09-18T01:10:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4338201600 bytes 2024-09-18T01:10:07.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4395678720 bytes 2024-09-18T01:10:07.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4476450816 bytes 2024-09-18T01:10:08.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4521076736 bytes 2024-09-18T01:10:08.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4543617024 bytes 2024-09-18T01:10:08.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4597839872 bytes 2024-09-18T01:10:08.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:10:08.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:10:08.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:10:08.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_5 2024-09-18T01:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 snapshot 128M 2024-09-18T01:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_5 128M 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_5 2024-09-18T01:10:08.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:10:08.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:08.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:10:08.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:08.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_5 snapshot 2024-09-18T01:10:08.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:08.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:08.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:10:08.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:08.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2024-09-18T01:10:09.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:10:09.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2024-09-18T01:10:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_5': 2024-09-18T01:10:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:10:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:10:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:10:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1251bd768daf 2024-09-18T01:10:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1251bd768daf 2024-09-18T01:10:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:10:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:10:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:10:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:10:09.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:08 2024 2024-09-18T01:10:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:08 2024 2024-09-18T01:10:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:08 2024 2024-09-18T01:10:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:10:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:10:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: abb2637a-7d4a-4f6e-a9ca-ee069dffe536 2024-09-18T01:10:09.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_5 2024-09-18T01:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:10:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=8 2024-09-18T01:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:10:09.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 619520 bytes 2024-09-18T01:10:09.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 19592192 bytes 2024-09-18T01:10:09.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 20221952 bytes 2024-09-18T01:10:09.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 74390528 bytes 2024-09-18T01:10:09.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 145603584 bytes 2024-09-18T01:10:10.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 201972736 bytes 2024-09-18T01:10:10.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 247919616 bytes 2024-09-18T01:10:10.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 306593792 bytes 2024-09-18T01:10:10.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 328220672 bytes 2024-09-18T01:10:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 363036672 bytes 2024-09-18T01:10:10.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 416720896 bytes 2024-09-18T01:10:10.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 460683264 bytes 2024-09-18T01:10:10.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 504039424 bytes 2024-09-18T01:10:11.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 600142848 bytes 2024-09-18T01:10:11.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 610898944 bytes 2024-09-18T01:10:11.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 699899904 bytes 2024-09-18T01:10:11.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 704007168 bytes 2024-09-18T01:10:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 726808576 bytes 2024-09-18T01:10:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 790179840 bytes 2024-09-18T01:10:11.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 861467648 bytes 2024-09-18T01:10:11.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 894480384 bytes 2024-09-18T01:10:12.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 958529536 bytes 2024-09-18T01:10:12.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 971453440 bytes 2024-09-18T01:10:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 994099200 bytes 2024-09-18T01:10:12.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1031729152 bytes 2024-09-18T01:10:12.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1081136128 bytes 2024-09-18T01:10:12.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1157282816 bytes 2024-09-18T01:10:12.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1196363776 bytes 2024-09-18T01:10:12.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1275807744 bytes 2024-09-18T01:10:12.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1286764544 bytes 2024-09-18T01:10:13.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1377719296 bytes 2024-09-18T01:10:13.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1429862400 bytes 2024-09-18T01:10:13.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1430468608 bytes 2024-09-18T01:10:13.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1454991360 bytes 2024-09-18T01:10:13.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1502915584 bytes 2024-09-18T01:10:13.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1556917248 bytes 2024-09-18T01:10:13.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1624345600 bytes 2024-09-18T01:10:13.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1660933120 bytes 2024-09-18T01:10:14.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1707099136 bytes 2024-09-18T01:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1747599360 bytes 2024-09-18T01:10:14.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1804175360 bytes 2024-09-18T01:10:14.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1860235264 bytes 2024-09-18T01:10:14.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1894823936 bytes 2024-09-18T01:10:14.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2002122752 bytes 2024-09-18T01:10:14.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2073594880 bytes 2024-09-18T01:10:14.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2097737728 bytes 2024-09-18T01:10:15.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2123419648 bytes 2024-09-18T01:10:15.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2156954624 bytes 2024-09-18T01:10:15.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2236835840 bytes 2024-09-18T01:10:15.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2301310976 bytes 2024-09-18T01:10:15.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2328524800 bytes 2024-09-18T01:10:15.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2330385408 bytes 2024-09-18T01:10:15.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2347900928 bytes 2024-09-18T01:10:15.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2394384384 bytes 2024-09-18T01:10:16.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2491641856 bytes 2024-09-18T01:10:16.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2509442048 bytes 2024-09-18T01:10:16.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2589985792 bytes 2024-09-18T01:10:16.396 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2605881344 bytes 2024-09-18T01:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2608251904 bytes 2024-09-18T01:10:16.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2633925632 bytes 2024-09-18T01:10:16.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2646406144 bytes 2024-09-18T01:10:16.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2683427840 bytes 2024-09-18T01:10:16.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2729846784 bytes 2024-09-18T01:10:16.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2751788032 bytes 2024-09-18T01:10:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2790609920 bytes 2024-09-18T01:10:17.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2853018624 bytes 2024-09-18T01:10:17.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:10:17.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_6 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 snapshot 128M 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:10:17.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:17.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_6 128M 2024-09-18T01:10:17.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:17.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:17.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:17.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_6 2024-09-18T01:10:17.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:10:17.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:17.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:10:17.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:17.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_6 snapshot 2024-09-18T01:10:17.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:17.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:17.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:10:17.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:17.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2024-09-18T01:10:18.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:10:18.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2024-09-18T01:10:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_6': 2024-09-18T01:10:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:10:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:10:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 125adc1ee7e0 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.125adc1ee7e0 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:17 2024 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:17 2024 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:17 2024 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: cbcb3575-14f3-490d-a7c4-517bdeb6f42a 2024-09-18T01:10:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:10:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:10:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_6 2024-09-18T01:10:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:10:18.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:10:18.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=7 2024-09-18T01:10:18.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:10:18.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:10:18.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 47481856 bytes 2024-09-18T01:10:18.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 86600704 bytes 2024-09-18T01:10:18.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 158797824 bytes 2024-09-18T01:10:18.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 217091072 bytes 2024-09-18T01:10:19.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 303129600 bytes 2024-09-18T01:10:19.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 347212800 bytes 2024-09-18T01:10:19.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 449521664 bytes 2024-09-18T01:10:19.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 484310016 bytes 2024-09-18T01:10:19.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 507356160 bytes 2024-09-18T01:10:19.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 579544064 bytes 2024-09-18T01:10:19.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 661884928 bytes 2024-09-18T01:10:20.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 735892480 bytes 2024-09-18T01:10:20.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 763223040 bytes 2024-09-18T01:10:20.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 785292288 bytes 2024-09-18T01:10:20.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 853263360 bytes 2024-09-18T01:10:20.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 924785664 bytes 2024-09-18T01:10:20.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 957726720 bytes 2024-09-18T01:10:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 975582208 bytes 2024-09-18T01:10:20.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 1024292864 bytes 2024-09-18T01:10:20.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 1071705088 bytes 2024-09-18T01:10:21.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1103151104 bytes 2024-09-18T01:10:21.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1196197888 bytes 2024-09-18T01:10:21.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1236453376 bytes 2024-09-18T01:10:21.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1266120704 bytes 2024-09-18T01:10:21.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1293461504 bytes 2024-09-18T01:10:21.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1347993600 bytes 2024-09-18T01:10:21.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1394172928 bytes 2024-09-18T01:10:21.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1473245184 bytes 2024-09-18T01:10:22.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1499597824 bytes 2024-09-18T01:10:22.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1585378304 bytes 2024-09-18T01:10:22.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1645009920 bytes 2024-09-18T01:10:22.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1654493184 bytes 2024-09-18T01:10:22.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1658773504 bytes 2024-09-18T01:10:22.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1685232640 bytes 2024-09-18T01:10:22.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1706368000 bytes 2024-09-18T01:10:22.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1757956096 bytes 2024-09-18T01:10:22.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1769066496 bytes 2024-09-18T01:10:23.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1791029248 bytes 2024-09-18T01:10:23.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1808604160 bytes 2024-09-18T01:10:23.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1827562496 bytes 2024-09-18T01:10:23.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1889282048 bytes 2024-09-18T01:10:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1938786304 bytes 2024-09-18T01:10:23.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1948905472 bytes 2024-09-18T01:10:23.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1961292800 bytes 2024-09-18T01:10:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1969628160 bytes 2024-09-18T01:10:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1977829376 bytes 2024-09-18T01:10:23.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2010932224 bytes 2024-09-18T01:10:24.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2064677888 bytes 2024-09-18T01:10:24.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2088153088 bytes 2024-09-18T01:10:24.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2130310144 bytes 2024-09-18T01:10:24.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2185401344 bytes 2024-09-18T01:10:24.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2242810880 bytes 2024-09-18T01:10:24.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2277582848 bytes 2024-09-18T01:10:24.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2331110400 bytes 2024-09-18T01:10:24.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2403429376 bytes 2024-09-18T01:10:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2422242304 bytes 2024-09-18T01:10:25.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2507398144 bytes 2024-09-18T01:10:25.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2541907968 bytes 2024-09-18T01:10:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2560190464 bytes 2024-09-18T01:10:25.380 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-18T01:10:25.381 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-18T01:10:25.381 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-18T01:10:25.381 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-18T01:10:25.381 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:10:25.383 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:10:25.380+0000 7f6c87957640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 42679) UID: 1000 2024-09-18T01:10:25.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:10:25.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_7 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 snapshot 128M 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:10:25.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_7 128M 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_7 2024-09-18T01:10:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:10:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:10:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_7 snapshot 2024-09-18T01:10:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:10:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:25.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2024-09-18T01:10:26.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:10:26.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2024-09-18T01:10:26.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_7': 2024-09-18T01:10:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:10:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:10:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:10:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12632e3d0d25 2024-09-18T01:10:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12632e3d0d25 2024-09-18T01:10:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:10:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:10:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:10:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:10:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:25 2024 2024-09-18T01:10:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:25 2024 2024-09-18T01:10:26.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:25 2024 2024-09-18T01:10:26.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:10:26.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:10:26.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 60a348dd-1555-4a46-89d2-8585c357735d 2024-09-18T01:10:26.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:10:26.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:10:26.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_7 2024-09-18T01:10:26.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:26.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:26.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:10:26.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:10:26.424 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=11 2024-09-18T01:10:26.424 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:10:26.424 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:10:26.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 8769536 bytes 2024-09-18T01:10:26.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 47381504 bytes 2024-09-18T01:10:26.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 115904512 bytes 2024-09-18T01:10:26.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 152503296 bytes 2024-09-18T01:10:26.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 169249792 bytes 2024-09-18T01:10:27.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 209427456 bytes 2024-09-18T01:10:27.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 261994496 bytes 2024-09-18T01:10:27.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 266889216 bytes 2024-09-18T01:10:27.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 335237120 bytes 2024-09-18T01:10:27.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 365888512 bytes 2024-09-18T01:10:27.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 395184128 bytes 2024-09-18T01:10:27.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 424028160 bytes 2024-09-18T01:10:27.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 483395584 bytes 2024-09-18T01:10:28.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 514625536 bytes 2024-09-18T01:10:28.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 557839360 bytes 2024-09-18T01:10:28.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 583325696 bytes 2024-09-18T01:10:28.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 651860992 bytes 2024-09-18T01:10:28.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 688550912 bytes 2024-09-18T01:10:28.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 716630016 bytes 2024-09-18T01:10:28.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 750730240 bytes 2024-09-18T01:10:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 860395520 bytes 2024-09-18T01:10:28.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 899952640 bytes 2024-09-18T01:10:29.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 973902848 bytes 2024-09-18T01:10:29.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1021079552 bytes 2024-09-18T01:10:29.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1090665472 bytes 2024-09-18T01:10:29.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1110948864 bytes 2024-09-18T01:10:29.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1174096896 bytes 2024-09-18T01:10:29.796 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1256940544 bytes 2024-09-18T01:10:29.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1349881856 bytes 2024-09-18T01:10:30.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1397612544 bytes 2024-09-18T01:10:30.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1443634176 bytes 2024-09-18T01:10:30.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1488186368 bytes 2024-09-18T01:10:30.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1520108544 bytes 2024-09-18T01:10:30.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1603032064 bytes 2024-09-18T01:10:30.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1620168704 bytes 2024-09-18T01:10:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1688587264 bytes 2024-09-18T01:10:30.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1715524608 bytes 2024-09-18T01:10:31.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1757069312 bytes 2024-09-18T01:10:31.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1791486976 bytes 2024-09-18T01:10:31.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1799936000 bytes 2024-09-18T01:10:31.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1841844224 bytes 2024-09-18T01:10:31.483 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-18T01:10:31.483 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-18T01:10:31.484 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-18T01:10:31.488 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-18T01:10:31.488 INFO:tasks.rbd_mirror_thrash:waiting for 119 secs before thrashing 2024-09-18T01:10:31.490 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1908585472 bytes 2024-09-18T01:10:31.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1926737920 bytes 2024-09-18T01:10:31.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1942681600 bytes 2024-09-18T01:10:31.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2025804800 bytes 2024-09-18T01:10:31.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2076768256 bytes 2024-09-18T01:10:32.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2083555328 bytes 2024-09-18T01:10:32.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2114593792 bytes 2024-09-18T01:10:32.272 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2121436160 bytes 2024-09-18T01:10:32.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2140280832 bytes 2024-09-18T01:10:32.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2165222400 bytes 2024-09-18T01:10:32.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2181660672 bytes 2024-09-18T01:10:32.658 INFO:tasks.ceph.cluster2.osd.1.smithi150.stderr:2024-09-18T01:10:32.656+0000 7f3bf08d2640 -1 reset not still connected to 0x55b6f1b1c820 2024-09-18T01:10:32.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2196487168 bytes 2024-09-18T01:10:32.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2199444480 bytes 2024-09-18T01:10:32.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2294550528 bytes 2024-09-18T01:10:32.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2295149568 bytes 2024-09-18T01:10:33.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2356383744 bytes 2024-09-18T01:10:33.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2378178560 bytes 2024-09-18T01:10:33.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2435280896 bytes 2024-09-18T01:10:33.471 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2476388352 bytes 2024-09-18T01:10:33.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2541702144 bytes 2024-09-18T01:10:33.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2617905152 bytes 2024-09-18T01:10:33.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2656266240 bytes 2024-09-18T01:10:34.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2718308352 bytes 2024-09-18T01:10:34.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2737017856 bytes 2024-09-18T01:10:34.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2791965696 bytes 2024-09-18T01:10:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2840013824 bytes 2024-09-18T01:10:34.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2919698432 bytes 2024-09-18T01:10:34.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2930135040 bytes 2024-09-18T01:10:34.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2969511936 bytes 2024-09-18T01:10:34.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3034734592 bytes 2024-09-18T01:10:34.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3036856320 bytes 2024-09-18T01:10:35.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3123526656 bytes 2024-09-18T01:10:35.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3168846848 bytes 2024-09-18T01:10:35.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3220530176 bytes 2024-09-18T01:10:35.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3246394368 bytes 2024-09-18T01:10:35.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3315454976 bytes 2024-09-18T01:10:35.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3349633024 bytes 2024-09-18T01:10:35.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3439758336 bytes 2024-09-18T01:10:35.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3440371712 bytes 2024-09-18T01:10:36.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3476852736 bytes 2024-09-18T01:10:36.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3503713280 bytes 2024-09-18T01:10:36.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3509429248 bytes 2024-09-18T01:10:36.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3568751616 bytes 2024-09-18T01:10:36.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3625936896 bytes 2024-09-18T01:10:36.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3654458368 bytes 2024-09-18T01:10:36.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3679529984 bytes 2024-09-18T01:10:36.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3721678848 bytes 2024-09-18T01:10:36.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3769227264 bytes 2024-09-18T01:10:37.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3853728768 bytes 2024-09-18T01:10:37.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3905047552 bytes 2024-09-18T01:10:37.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3966498816 bytes 2024-09-18T01:10:37.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:10:37.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:10:37.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:10:37.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:37.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:10:37.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_8 2024-09-18T01:10:37.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 snapshot 128M 2024-09-18T01:10:37.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:10:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:10:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:37.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_8 128M 2024-09-18T01:10:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:10:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:10:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:10:37.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:10:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:37.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_8 2024-09-18T01:10:37.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:10:37.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:37.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:10:37.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:37.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_8 snapshot 2024-09-18T01:10:37.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:37.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:37.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:10:37.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:37.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2024-09-18T01:10:38.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:10:38.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2024-09-18T01:10:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_8': 2024-09-18T01:10:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:10:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:10:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:10:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12735302263d 2024-09-18T01:10:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12735302263d 2024-09-18T01:10:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:10:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:10:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:10:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:10:38.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:37 2024 2024-09-18T01:10:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:37 2024 2024-09-18T01:10:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:37 2024 2024-09-18T01:10:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:10:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:10:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c8657da4-764c-45a1-9898-eb5d4ab62f4f 2024-09-18T01:10:38.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:10:38.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:10:38.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_8 2024-09-18T01:10:38.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:38.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:38.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:10:38.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:10:38.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=7 2024-09-18T01:10:38.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:10:38.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:10:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 41866240 bytes 2024-09-18T01:10:38.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 90629120 bytes 2024-09-18T01:10:38.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 134509568 bytes 2024-09-18T01:10:38.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 189037568 bytes 2024-09-18T01:10:39.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 237368320 bytes 2024-09-18T01:10:39.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 279795712 bytes 2024-09-18T01:10:39.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 307827712 bytes 2024-09-18T01:10:39.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 309138432 bytes 2024-09-18T01:10:39.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 347174912 bytes 2024-09-18T01:10:39.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 409806848 bytes 2024-09-18T01:10:39.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 433378304 bytes 2024-09-18T01:10:39.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 482767872 bytes 2024-09-18T01:10:39.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 529290240 bytes 2024-09-18T01:10:40.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 550507520 bytes 2024-09-18T01:10:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 601954304 bytes 2024-09-18T01:10:40.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 649700352 bytes 2024-09-18T01:10:40.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 696325120 bytes 2024-09-18T01:10:40.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 723718144 bytes 2024-09-18T01:10:40.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 851003392 bytes 2024-09-18T01:10:40.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 900169728 bytes 2024-09-18T01:10:40.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 953939968 bytes 2024-09-18T01:10:41.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1031775232 bytes 2024-09-18T01:10:41.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1110652928 bytes 2024-09-18T01:10:41.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1137874944 bytes 2024-09-18T01:10:41.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1158440960 bytes 2024-09-18T01:10:41.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1243750400 bytes 2024-09-18T01:10:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1327970304 bytes 2024-09-18T01:10:41.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1344226304 bytes 2024-09-18T01:10:41.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1361185792 bytes 2024-09-18T01:10:42.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1378514944 bytes 2024-09-18T01:10:42.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1435718656 bytes 2024-09-18T01:10:42.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1475727360 bytes 2024-09-18T01:10:42.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1526311936 bytes 2024-09-18T01:10:42.425 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.2584119984863287) > weight (0.25) 2024-09-18T01:10:42.425 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8550881275484613) > weight (0.25) 2024-09-18T01:10:42.425 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.41221394215739093) > weight (0.25) 2024-09-18T01:10:42.425 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.3057648759191184) > weight (0.25) 2024-09-18T01:10:42.425 INFO:tasks.rbd_mirror_thrash:waiting for 91 secs before thrashing 2024-09-18T01:10:42.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1595323392 bytes 2024-09-18T01:10:42.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1672811520 bytes 2024-09-18T01:10:42.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1721085952 bytes 2024-09-18T01:10:42.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1782068224 bytes 2024-09-18T01:10:43.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1807705088 bytes 2024-09-18T01:10:43.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1897146368 bytes 2024-09-18T01:10:43.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1946516480 bytes 2024-09-18T01:10:43.423 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1967371264 bytes 2024-09-18T01:10:43.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 2010536960 bytes 2024-09-18T01:10:43.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2055473152 bytes 2024-09-18T01:10:43.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2145736704 bytes 2024-09-18T01:10:43.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2159512576 bytes 2024-09-18T01:10:44.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2202398720 bytes 2024-09-18T01:10:44.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2249699328 bytes 2024-09-18T01:10:44.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2343803904 bytes 2024-09-18T01:10:44.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2355407872 bytes 2024-09-18T01:10:44.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2431924224 bytes 2024-09-18T01:10:44.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2467743744 bytes 2024-09-18T01:10:44.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2528626688 bytes 2024-09-18T01:10:44.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2582286336 bytes 2024-09-18T01:10:45.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2674889728 bytes 2024-09-18T01:10:45.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2697658368 bytes 2024-09-18T01:10:45.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2719596544 bytes 2024-09-18T01:10:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:10:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:10:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:10:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:10:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_9 2024-09-18T01:10:45.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 snapshot 128M 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_9 128M 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:45.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:10:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:10:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:10:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:10:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:45.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_9 2024-09-18T01:10:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:10:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:10:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_9 snapshot 2024-09-18T01:10:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:10:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:45.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2024-09-18T01:10:46.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:10:46.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2024-09-18T01:10:46.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_9': 2024-09-18T01:10:46.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:10:46.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:10:46.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:10:46.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 127b648c7850 2024-09-18T01:10:46.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.127b648c7850 2024-09-18T01:10:46.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:10:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:10:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:10:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:10:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:45 2024 2024-09-18T01:10:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:45 2024 2024-09-18T01:10:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:45 2024 2024-09-18T01:10:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:10:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:10:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 28c48b31-f556-4f13-90b4-b7bc04a088dd 2024-09-18T01:10:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:10:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:10:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_9 2024-09-18T01:10:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:10:46.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:10:46.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=10 2024-09-18T01:10:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:10:46.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:10:46.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 65667072 bytes 2024-09-18T01:10:46.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 101288960 bytes 2024-09-18T01:10:46.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 141309952 bytes 2024-09-18T01:10:46.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 141917184 bytes 2024-09-18T01:10:47.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 200903680 bytes 2024-09-18T01:10:47.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 243393536 bytes 2024-09-18T01:10:47.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 280422400 bytes 2024-09-18T01:10:47.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 334499840 bytes 2024-09-18T01:10:47.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 348126208 bytes 2024-09-18T01:10:47.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 412485632 bytes 2024-09-18T01:10:47.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 481537024 bytes 2024-09-18T01:10:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 579745792 bytes 2024-09-18T01:10:48.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 650212352 bytes 2024-09-18T01:10:48.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 676307968 bytes 2024-09-18T01:10:48.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 749607936 bytes 2024-09-18T01:10:48.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 786990080 bytes 2024-09-18T01:10:48.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 882211840 bytes 2024-09-18T01:10:48.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 924446720 bytes 2024-09-18T01:10:48.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 951621632 bytes 2024-09-18T01:10:49.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 952231936 bytes 2024-09-18T01:10:49.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1022828544 bytes 2024-09-18T01:10:49.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1085725696 bytes 2024-09-18T01:10:49.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1102341120 bytes 2024-09-18T01:10:49.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1118423040 bytes 2024-09-18T01:10:49.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1173365760 bytes 2024-09-18T01:10:49.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1213637632 bytes 2024-09-18T01:10:49.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1260472320 bytes 2024-09-18T01:10:49.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1324054528 bytes 2024-09-18T01:10:50.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1420973056 bytes 2024-09-18T01:10:50.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1519583232 bytes 2024-09-18T01:10:50.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1545190400 bytes 2024-09-18T01:10:50.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1590386688 bytes 2024-09-18T01:10:50.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1608432640 bytes 2024-09-18T01:10:50.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1645442048 bytes 2024-09-18T01:10:50.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1705030656 bytes 2024-09-18T01:10:50.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1729400832 bytes 2024-09-18T01:10:51.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1788762112 bytes 2024-09-18T01:10:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1828753408 bytes 2024-09-18T01:10:51.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1868705792 bytes 2024-09-18T01:10:51.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1889631232 bytes 2024-09-18T01:10:51.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1925093376 bytes 2024-09-18T01:10:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1964096512 bytes 2024-09-18T01:10:51.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2007030784 bytes 2024-09-18T01:10:51.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2110777344 bytes 2024-09-18T01:10:52.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2151036928 bytes 2024-09-18T01:10:52.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2166026240 bytes 2024-09-18T01:10:52.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2206747648 bytes 2024-09-18T01:10:52.423 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2216396800 bytes 2024-09-18T01:10:52.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2243949568 bytes 2024-09-18T01:10:52.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2277787648 bytes 2024-09-18T01:10:52.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2363184128 bytes 2024-09-18T01:10:52.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2407904256 bytes 2024-09-18T01:10:53.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2435001344 bytes 2024-09-18T01:10:53.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2480119808 bytes 2024-09-18T01:10:53.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2540148736 bytes 2024-09-18T01:10:53.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2564611072 bytes 2024-09-18T01:10:53.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2645190656 bytes 2024-09-18T01:10:53.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2702601216 bytes 2024-09-18T01:10:53.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2765433856 bytes 2024-09-18T01:10:53.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2809638912 bytes 2024-09-18T01:10:54.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2860752896 bytes 2024-09-18T01:10:54.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2938271744 bytes 2024-09-18T01:10:54.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2961593344 bytes 2024-09-18T01:10:54.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2970263552 bytes 2024-09-18T01:10:54.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 3061876736 bytes 2024-09-18T01:10:54.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 3118682112 bytes 2024-09-18T01:10:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3172975616 bytes 2024-09-18T01:10:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3195473920 bytes 2024-09-18T01:10:55.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3228865536 bytes 2024-09-18T01:10:55.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3321496576 bytes 2024-09-18T01:10:55.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3322106880 bytes 2024-09-18T01:10:55.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3322712064 bytes 2024-09-18T01:10:55.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3344126976 bytes 2024-09-18T01:10:55.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3396495360 bytes 2024-09-18T01:10:55.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3433199616 bytes 2024-09-18T01:10:55.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3438567424 bytes 2024-09-18T01:10:56.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3469278208 bytes 2024-09-18T01:10:56.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3498626048 bytes 2024-09-18T01:10:56.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3532795904 bytes 2024-09-18T01:10:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3573916672 bytes 2024-09-18T01:10:56.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3589702656 bytes 2024-09-18T01:10:56.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:10:56.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:10:56.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_10 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 snapshot 128M 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:56.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_10 128M 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:10:56.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_10 2024-09-18T01:10:56.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:10:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:10:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:10:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_10 snapshot 2024-09-18T01:10:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:10:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:10:56.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2024-09-18T01:10:57.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:10:57.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2024-09-18T01:10:57.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_10': 2024-09-18T01:10:57.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:10:57.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:10:57.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:10:57.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12836023a0d4 2024-09-18T01:10:57.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12836023a0d4 2024-09-18T01:10:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:10:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:10:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:10:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:10:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:56 2024 2024-09-18T01:10:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:56 2024 2024-09-18T01:10:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:56 2024 2024-09-18T01:10:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:10:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:10:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 0a5efa51-897f-43ff-9dc4-ca67a0c87f5d 2024-09-18T01:10:57.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:10:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:10:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_10 2024-09-18T01:10:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:10:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:10:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:10:57.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:10:57.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:10:57.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:10:57.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:10:57.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 72594432 bytes 2024-09-18T01:10:57.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 73210880 bytes 2024-09-18T01:10:57.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 130203648 bytes 2024-09-18T01:10:58.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 174259200 bytes 2024-09-18T01:10:58.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 184179712 bytes 2024-09-18T01:10:58.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 251109376 bytes 2024-09-18T01:10:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 275067904 bytes 2024-09-18T01:10:58.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 321421312 bytes 2024-09-18T01:10:58.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 397639680 bytes 2024-09-18T01:10:58.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 478118912 bytes 2024-09-18T01:10:58.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 581233664 bytes 2024-09-18T01:10:59.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 639614976 bytes 2024-09-18T01:10:59.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 688025600 bytes 2024-09-18T01:10:59.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 737942528 bytes 2024-09-18T01:10:59.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 803694592 bytes 2024-09-18T01:10:59.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 897960960 bytes 2024-09-18T01:10:59.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 921284608 bytes 2024-09-18T01:10:59.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 944124928 bytes 2024-09-18T01:10:59.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 958675968 bytes 2024-09-18T01:11:00.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 1012986880 bytes 2024-09-18T01:11:00.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1120328704 bytes 2024-09-18T01:11:00.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1146409984 bytes 2024-09-18T01:11:00.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1147025408 bytes 2024-09-18T01:11:00.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1207188480 bytes 2024-09-18T01:11:00.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1247707136 bytes 2024-09-18T01:11:00.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1287379968 bytes 2024-09-18T01:11:00.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1333679104 bytes 2024-09-18T01:11:00.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1343456256 bytes 2024-09-18T01:11:01.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1457458176 bytes 2024-09-18T01:11:01.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1476951040 bytes 2024-09-18T01:11:01.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1596460032 bytes 2024-09-18T01:11:01.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1648533504 bytes 2024-09-18T01:11:01.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1704587264 bytes 2024-09-18T01:11:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1735974912 bytes 2024-09-18T01:11:01.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1757786112 bytes 2024-09-18T01:11:01.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1784884224 bytes 2024-09-18T01:11:02.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1844551680 bytes 2024-09-18T01:11:02.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1856225280 bytes 2024-09-18T01:11:02.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1922604032 bytes 2024-09-18T01:11:02.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1957307392 bytes 2024-09-18T01:11:02.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 2009399296 bytes 2024-09-18T01:11:02.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 2032776192 bytes 2024-09-18T01:11:02.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2100638720 bytes 2024-09-18T01:11:02.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2145655808 bytes 2024-09-18T01:11:03.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2233752576 bytes 2024-09-18T01:11:03.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2282973184 bytes 2024-09-18T01:11:03.363 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2339570688 bytes 2024-09-18T01:11:03.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2385909760 bytes 2024-09-18T01:11:03.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2445691904 bytes 2024-09-18T01:11:03.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2489116672 bytes 2024-09-18T01:11:03.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2532247552 bytes 2024-09-18T01:11:03.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2558479360 bytes 2024-09-18T01:11:04.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2606203904 bytes 2024-09-18T01:11:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2655277056 bytes 2024-09-18T01:11:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2686426112 bytes 2024-09-18T01:11:04.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2734529536 bytes 2024-09-18T01:11:04.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2838724608 bytes 2024-09-18T01:11:04.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2865073152 bytes 2024-09-18T01:11:04.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2949519360 bytes 2024-09-18T01:11:04.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2955906048 bytes 2024-09-18T01:11:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2976110592 bytes 2024-09-18T01:11:05.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2979076096 bytes 2024-09-18T01:11:05.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2989599744 bytes 2024-09-18T01:11:05.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 3026638848 bytes 2024-09-18T01:11:05.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 3090813952 bytes 2024-09-18T01:11:05.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 3146189824 bytes 2024-09-18T01:11:05.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3196466176 bytes 2024-09-18T01:11:05.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3238887424 bytes 2024-09-18T01:11:06.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3290506240 bytes 2024-09-18T01:11:06.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3360061440 bytes 2024-09-18T01:11:06.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3386696704 bytes 2024-09-18T01:11:06.424 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3473024000 bytes 2024-09-18T01:11:06.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3508291584 bytes 2024-09-18T01:11:06.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3532871680 bytes 2024-09-18T01:11:06.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3589293056 bytes 2024-09-18T01:11:06.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3617117184 bytes 2024-09-18T01:11:06.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3634668544 bytes 2024-09-18T01:11:07.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3659947008 bytes 2024-09-18T01:11:07.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3712205824 bytes 2024-09-18T01:11:07.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3764910080 bytes 2024-09-18T01:11:07.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3850871808 bytes 2024-09-18T01:11:07.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3884347392 bytes 2024-09-18T01:11:07.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3926577152 bytes 2024-09-18T01:11:07.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3951342592 bytes 2024-09-18T01:11:07.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3983440896 bytes 2024-09-18T01:11:08.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 4041742336 bytes 2024-09-18T01:11:08.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 4089046016 bytes 2024-09-18T01:11:08.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 4119648256 bytes 2024-09-18T01:11:08.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 4208900096 bytes 2024-09-18T01:11:08.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 4278469632 bytes 2024-09-18T01:11:08.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 4357196800 bytes 2024-09-18T01:11:08.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4410355712 bytes 2024-09-18T01:11:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4449967104 bytes 2024-09-18T01:11:09.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4520862720 bytes 2024-09-18T01:11:09.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4573126656 bytes 2024-09-18T01:11:09.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4637714432 bytes 2024-09-18T01:11:09.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4728534016 bytes 2024-09-18T01:11:09.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4740401152 bytes 2024-09-18T01:11:09.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4838711296 bytes 2024-09-18T01:11:09.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4873623552 bytes 2024-09-18T01:11:10.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4911367168 bytes 2024-09-18T01:11:10.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4937802752 bytes 2024-09-18T01:11:10.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4966425600 bytes 2024-09-18T01:11:10.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4997345280 bytes 2024-09-18T01:11:10.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 5024429056 bytes 2024-09-18T01:11:10.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 5037406208 bytes 2024-09-18T01:11:10.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 5040388096 bytes 2024-09-18T01:11:10.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 5065398272 bytes 2024-09-18T01:11:10.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 5131239424 bytes 2024-09-18T01:11:11.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 5154600960 bytes 2024-09-18T01:11:11.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11100 IOs, 5234437120 bytes 2024-09-18T01:11:11.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11200 IOs, 5270614016 bytes 2024-09-18T01:11:11.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11300 IOs, 5349170176 bytes 2024-09-18T01:11:11.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:11:11.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:11:11.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:11:11.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:11:11.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:11:11.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_11 2024-09-18T01:11:11.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 snapshot 128M 2024-09-18T01:11:11.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_11 128M 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:11.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:11:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:11:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:11:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:11:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_11 2024-09-18T01:11:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:11:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:11:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:11:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_11 snapshot 2024-09-18T01:11:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:11.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:11.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:11:11.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:11:11.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_11 snapshot 2024-09-18T01:11:12.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:11:12.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_11 2024-09-18T01:11:12.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_11': 2024-09-18T01:11:12.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 128c43fe6c4b 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.128c43fe6c4b 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:11:11 2024 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:11:11 2024 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:11:11 2024 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:11:12.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:11:12.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 067a8b9f-a615-4968-8d24-956a8b2892a6 2024-09-18T01:11:12.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:11:12.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:11:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_11 2024-09-18T01:11:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:11:12.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:11:12.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=10 2024-09-18T01:11:12.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:11:12.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:11:12.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 44322816 bytes 2024-09-18T01:11:12.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 79569920 bytes 2024-09-18T01:11:12.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 145700864 bytes 2024-09-18T01:11:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 158367744 bytes 2024-09-18T01:11:13.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 202413056 bytes 2024-09-18T01:11:13.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 221918208 bytes 2024-09-18T01:11:13.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 288506880 bytes 2024-09-18T01:11:13.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 345214976 bytes 2024-09-18T01:11:13.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 437386240 bytes 2024-09-18T01:11:13.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 458048512 bytes 2024-09-18T01:11:13.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 488741888 bytes 2024-09-18T01:11:14.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 545265664 bytes 2024-09-18T01:11:14.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 599413760 bytes 2024-09-18T01:11:14.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 623056896 bytes 2024-09-18T01:11:14.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 681307136 bytes 2024-09-18T01:11:14.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 700789760 bytes 2024-09-18T01:11:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 795980800 bytes 2024-09-18T01:11:14.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 837833728 bytes 2024-09-18T01:11:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 867175424 bytes 2024-09-18T01:11:15.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 938337280 bytes 2024-09-18T01:11:15.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1014501376 bytes 2024-09-18T01:11:15.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1051683840 bytes 2024-09-18T01:11:15.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1068081152 bytes 2024-09-18T01:11:15.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1121903616 bytes 2024-09-18T01:11:15.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1172260864 bytes 2024-09-18T01:11:15.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1219863552 bytes 2024-09-18T01:11:15.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1245507584 bytes 2024-09-18T01:11:16.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1280146432 bytes 2024-09-18T01:11:16.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1353088000 bytes 2024-09-18T01:11:16.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1456748544 bytes 2024-09-18T01:11:16.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1502994432 bytes 2024-09-18T01:11:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1549496320 bytes 2024-09-18T01:11:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1633646592 bytes 2024-09-18T01:11:16.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1703366656 bytes 2024-09-18T01:11:17.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1724106752 bytes 2024-09-18T01:11:17.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1778052096 bytes 2024-09-18T01:11:17.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1813495808 bytes 2024-09-18T01:11:17.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1814103040 bytes 2024-09-18T01:11:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1862575104 bytes 2024-09-18T01:11:17.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1959467008 bytes 2024-09-18T01:11:17.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 2022000640 bytes 2024-09-18T01:11:17.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 2026839040 bytes 2024-09-18T01:11:18.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2055303168 bytes 2024-09-18T01:11:18.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2093949952 bytes 2024-09-18T01:11:18.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2117500928 bytes 2024-09-18T01:11:18.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2153960448 bytes 2024-09-18T01:11:18.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2154582016 bytes 2024-09-18T01:11:18.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2240125952 bytes 2024-09-18T01:11:18.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2285961216 bytes 2024-09-18T01:11:18.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2337498112 bytes 2024-09-18T01:11:18.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2359235584 bytes 2024-09-18T01:11:19.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2404548608 bytes 2024-09-18T01:11:19.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2420911104 bytes 2024-09-18T01:11:19.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2443198464 bytes 2024-09-18T01:11:19.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2516403200 bytes 2024-09-18T01:11:19.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2557082624 bytes 2024-09-18T01:11:19.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2573997056 bytes 2024-09-18T01:11:19.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2619237376 bytes 2024-09-18T01:11:19.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2637147136 bytes 2024-09-18T01:11:19.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2679131136 bytes 2024-09-18T01:11:20.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2698133504 bytes 2024-09-18T01:11:20.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2807524352 bytes 2024-09-18T01:11:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2865699840 bytes 2024-09-18T01:11:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2866315264 bytes 2024-09-18T01:11:20.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2896474112 bytes 2024-09-18T01:11:20.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2913730560 bytes 2024-09-18T01:11:20.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2949406720 bytes 2024-09-18T01:11:20.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2984958976 bytes 2024-09-18T01:11:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3039393792 bytes 2024-09-18T01:11:21.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3069104128 bytes 2024-09-18T01:11:21.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3113269248 bytes 2024-09-18T01:11:21.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3144867840 bytes 2024-09-18T01:11:21.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3161099264 bytes 2024-09-18T01:11:21.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3193852928 bytes 2024-09-18T01:11:21.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3225895936 bytes 2024-09-18T01:11:21.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3271595008 bytes 2024-09-18T01:11:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3272197120 bytes 2024-09-18T01:11:21.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3292620800 bytes 2024-09-18T01:11:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3344074752 bytes 2024-09-18T01:11:22.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3373322240 bytes 2024-09-18T01:11:22.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3376171008 bytes 2024-09-18T01:11:22.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3406942208 bytes 2024-09-18T01:11:22.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3431549952 bytes 2024-09-18T01:11:22.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_12 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 snapshot 128M 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:11:22.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_12 128M 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:11:22.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:22.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_12 2024-09-18T01:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_12 snapshot 2024-09-18T01:11:22.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:22.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:22.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:11:22.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:11:22.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_12 snapshot 2024-09-18T01:11:23.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:11:23.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_12 2024-09-18T01:11:23.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_12': 2024-09-18T01:11:23.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1294e8c6d37c 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1294e8c6d37c 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:11:22 2024 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:11:22 2024 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:11:22 2024 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:11:23.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:11:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: b8fdbcd4-3f49-41c6-90b8-318c1f5ef36b 2024-09-18T01:11:23.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:11:23.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:11:23.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_12 2024-09-18T01:11:23.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:23.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:23.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:11:23.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:11:23.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=11 2024-09-18T01:11:23.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:11:23.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:11:23.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 75457536 bytes 2024-09-18T01:11:23.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 135686144 bytes 2024-09-18T01:11:23.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 162657280 bytes 2024-09-18T01:11:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 206064640 bytes 2024-09-18T01:11:24.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 318738432 bytes 2024-09-18T01:11:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 319362048 bytes 2024-09-18T01:11:24.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 393791488 bytes 2024-09-18T01:11:24.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 402419712 bytes 2024-09-18T01:11:24.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 424321024 bytes 2024-09-18T01:11:24.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 514195456 bytes 2024-09-18T01:11:24.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 521288704 bytes 2024-09-18T01:11:25.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 626309120 bytes 2024-09-18T01:11:25.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 673927168 bytes 2024-09-18T01:11:25.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 695922688 bytes 2024-09-18T01:11:25.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 742824960 bytes 2024-09-18T01:11:25.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 761493504 bytes 2024-09-18T01:11:25.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 822335488 bytes 2024-09-18T01:11:25.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 865818624 bytes 2024-09-18T01:11:25.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 911108096 bytes 2024-09-18T01:11:26.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 988558336 bytes 2024-09-18T01:11:26.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1021615104 bytes 2024-09-18T01:11:26.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1033212928 bytes 2024-09-18T01:11:26.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1101097984 bytes 2024-09-18T01:11:26.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1153993728 bytes 2024-09-18T01:11:26.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1206069248 bytes 2024-09-18T01:11:26.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1239014400 bytes 2024-09-18T01:11:26.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1278499840 bytes 2024-09-18T01:11:27.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1331570688 bytes 2024-09-18T01:11:27.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1373526016 bytes 2024-09-18T01:11:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1398139904 bytes 2024-09-18T01:11:27.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1433500672 bytes 2024-09-18T01:11:27.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1464992768 bytes 2024-09-18T01:11:27.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1492353024 bytes 2024-09-18T01:11:27.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1593385984 bytes 2024-09-18T01:11:27.843 INFO:tasks.ceph.cluster2.osd.0.smithi150.stderr:2024-09-18T01:11:27.842+0000 7f8980c28640 -1 reset not still connected to 0x55b500bed790 2024-09-18T01:11:27.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1607663616 bytes 2024-09-18T01:11:28.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1673876480 bytes 2024-09-18T01:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1788957696 bytes 2024-09-18T01:11:28.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1838755840 bytes 2024-09-18T01:11:28.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1863177216 bytes 2024-09-18T01:11:28.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1966242816 bytes 2024-09-18T01:11:28.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1997018112 bytes 2024-09-18T01:11:28.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 2043787264 bytes 2024-09-18T01:11:28.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2060453888 bytes 2024-09-18T01:11:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2094917632 bytes 2024-09-18T01:11:29.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2136207360 bytes 2024-09-18T01:11:29.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2161857536 bytes 2024-09-18T01:11:29.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2207586304 bytes 2024-09-18T01:11:29.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2268019712 bytes 2024-09-18T01:11:29.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2290701312 bytes 2024-09-18T01:11:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2348279808 bytes 2024-09-18T01:11:29.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2381312000 bytes 2024-09-18T01:11:29.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2399458304 bytes 2024-09-18T01:11:30.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2473537536 bytes 2024-09-18T01:11:30.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2491215872 bytes 2024-09-18T01:11:30.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2544995328 bytes 2024-09-18T01:11:30.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2547363840 bytes 2024-09-18T01:11:30.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2588733440 bytes 2024-09-18T01:11:30.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2649032704 bytes 2024-09-18T01:11:30.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2663305216 bytes 2024-09-18T01:11:31.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2716749824 bytes 2024-09-18T01:11:31.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2718743552 bytes 2024-09-18T01:11:31.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2785169408 bytes 2024-09-18T01:11:31.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2785779712 bytes 2024-09-18T01:11:31.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2786374656 bytes 2024-09-18T01:11:31.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2871655424 bytes 2024-09-18T01:11:31.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2896907264 bytes 2024-09-18T01:11:31.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2911688704 bytes 2024-09-18T01:11:31.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2934895616 bytes 2024-09-18T01:11:32.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3034947584 bytes 2024-09-18T01:11:32.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3035552768 bytes 2024-09-18T01:11:32.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3037176832 bytes 2024-09-18T01:11:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3117499392 bytes 2024-09-18T01:11:32.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3234946048 bytes 2024-09-18T01:11:32.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3257656320 bytes 2024-09-18T01:11:32.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3296166912 bytes 2024-09-18T01:11:32.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3359446016 bytes 2024-09-18T01:11:33.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3391700992 bytes 2024-09-18T01:11:33.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3433208832 bytes 2024-09-18T01:11:33.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3533852672 bytes 2024-09-18T01:11:33.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3605477376 bytes 2024-09-18T01:11:33.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3656607744 bytes 2024-09-18T01:11:33.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3681244160 bytes 2024-09-18T01:11:33.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3773346816 bytes 2024-09-18T01:11:34.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3797730304 bytes 2024-09-18T01:11:34.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3804741632 bytes 2024-09-18T01:11:34.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3850192896 bytes 2024-09-18T01:11:34.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3850778624 bytes 2024-09-18T01:11:34.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3911364608 bytes 2024-09-18T01:11:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:11:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:11:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:11:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:11:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:11:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_13 2024-09-18T01:11:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 snapshot 128M 2024-09-18T01:11:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:34.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_13 128M 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:34.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:11:34.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:11:34.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:11:34.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:34.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_13 2024-09-18T01:11:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:11:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:11:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:11:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_13 snapshot 2024-09-18T01:11:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:34.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:11:34.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:11:34.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_13 snapshot 2024-09-18T01:11:35.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:11:35.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_13 2024-09-18T01:11:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_13': 2024-09-18T01:11:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:11:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:11:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:11:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 129def2db538 2024-09-18T01:11:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.129def2db538 2024-09-18T01:11:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:11:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:11:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:11:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:11:35.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:11:34 2024 2024-09-18T01:11:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:11:34 2024 2024-09-18T01:11:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:11:34 2024 2024-09-18T01:11:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:11:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:11:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: a60dcf1c-e06a-4d99-b676-d35f298225a4 2024-09-18T01:11:35.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:11:35.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:11:35.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_13 2024-09-18T01:11:35.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:35.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:35.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:11:35.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:11:35.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=5 2024-09-18T01:11:35.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:11:35.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:11:35.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 45779968 bytes 2024-09-18T01:11:35.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 91158528 bytes 2024-09-18T01:11:35.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 170468352 bytes 2024-09-18T01:11:36.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 222664704 bytes 2024-09-18T01:11:36.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 262689792 bytes 2024-09-18T01:11:36.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 301386752 bytes 2024-09-18T01:11:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 342262784 bytes 2024-09-18T01:11:36.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 394768384 bytes 2024-09-18T01:11:36.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 422195200 bytes 2024-09-18T01:11:36.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 492624896 bytes 2024-09-18T01:11:37.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 533931008 bytes 2024-09-18T01:11:37.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 573333504 bytes 2024-09-18T01:11:37.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 623119360 bytes 2024-09-18T01:11:37.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 648008704 bytes 2024-09-18T01:11:37.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 684007424 bytes 2024-09-18T01:11:37.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 709364736 bytes 2024-09-18T01:11:37.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 720781312 bytes 2024-09-18T01:11:37.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 750356480 bytes 2024-09-18T01:11:37.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 789571584 bytes 2024-09-18T01:11:38.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 790182912 bytes 2024-09-18T01:11:38.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 806031360 bytes 2024-09-18T01:11:38.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 926841856 bytes 2024-09-18T01:11:38.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 945986560 bytes 2024-09-18T01:11:38.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 966590464 bytes 2024-09-18T01:11:38.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 967209984 bytes 2024-09-18T01:11:38.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 978959360 bytes 2024-09-18T01:11:38.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1057388544 bytes 2024-09-18T01:11:39.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1118289920 bytes 2024-09-18T01:11:39.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1173340160 bytes 2024-09-18T01:11:39.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1239065600 bytes 2024-09-18T01:11:39.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1297158144 bytes 2024-09-18T01:11:39.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1343201280 bytes 2024-09-18T01:11:39.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1390092288 bytes 2024-09-18T01:11:39.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1433307136 bytes 2024-09-18T01:11:39.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1454194688 bytes 2024-09-18T01:11:40.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1480798208 bytes 2024-09-18T01:11:40.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1512091648 bytes 2024-09-18T01:11:40.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1545431040 bytes 2024-09-18T01:11:40.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1546037248 bytes 2024-09-18T01:11:40.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1553379328 bytes 2024-09-18T01:11:40.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1568859136 bytes 2024-09-18T01:11:40.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:11:40.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:11:40.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:11:40.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:11:40.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_14 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 snapshot 128M 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_14 128M 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:40.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:11:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:11:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:11:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:11:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:40.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_14 2024-09-18T01:11:40.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:11:40.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:40.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:11:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:11:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_14 snapshot 2024-09-18T01:11:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:11:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:11:40.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_14 snapshot 2024-09-18T01:11:41.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:11:41.528 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_14 2024-09-18T01:11:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_14': 2024-09-18T01:11:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12a5ac43bf6b 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12a5ac43bf6b 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:11:40 2024 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:11:40 2024 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:11:40 2024 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:11:41.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:11:41.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 6c6c64a2-50e5-44f2-8703-2da1463fc540 2024-09-18T01:11:41.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:11:41.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:11:41.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_14 2024-09-18T01:11:41.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:41.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:41.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:11:41.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:11:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=13 2024-09-18T01:11:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:11:41.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:11:41.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 628736 bytes 2024-09-18T01:11:41.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 75420672 bytes 2024-09-18T01:11:41.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 88307712 bytes 2024-09-18T01:11:42.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 125242368 bytes 2024-09-18T01:11:42.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 159261696 bytes 2024-09-18T01:11:42.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 174354432 bytes 2024-09-18T01:11:42.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 204766208 bytes 2024-09-18T01:11:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 278547456 bytes 2024-09-18T01:11:42.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 284760064 bytes 2024-09-18T01:11:42.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 324885504 bytes 2024-09-18T01:11:42.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 369087488 bytes 2024-09-18T01:11:42.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 382861312 bytes 2024-09-18T01:11:43.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 405346304 bytes 2024-09-18T01:11:43.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 464433152 bytes 2024-09-18T01:11:43.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 483319808 bytes 2024-09-18T01:11:43.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 505872384 bytes 2024-09-18T01:11:43.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 547517440 bytes 2024-09-18T01:11:43.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 609804288 bytes 2024-09-18T01:11:43.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 661915648 bytes 2024-09-18T01:11:43.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 721195008 bytes 2024-09-18T01:11:44.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 787494912 bytes 2024-09-18T01:11:44.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 818857984 bytes 2024-09-18T01:11:44.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 845239296 bytes 2024-09-18T01:11:44.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 871098368 bytes 2024-09-18T01:11:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 968660992 bytes 2024-09-18T01:11:44.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1040527360 bytes 2024-09-18T01:11:44.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1105386496 bytes 2024-09-18T01:11:44.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1127812096 bytes 2024-09-18T01:11:45.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1175729152 bytes 2024-09-18T01:11:45.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1225059328 bytes 2024-09-18T01:11:45.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1260140544 bytes 2024-09-18T01:11:45.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1262345216 bytes 2024-09-18T01:11:45.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1363762176 bytes 2024-09-18T01:11:45.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1420612608 bytes 2024-09-18T01:11:45.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1443543040 bytes 2024-09-18T01:11:45.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1486536704 bytes 2024-09-18T01:11:46.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1494704128 bytes 2024-09-18T01:11:46.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1539846144 bytes 2024-09-18T01:11:46.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1570028544 bytes 2024-09-18T01:11:46.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1588763648 bytes 2024-09-18T01:11:46.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1620146176 bytes 2024-09-18T01:11:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1654821888 bytes 2024-09-18T01:11:46.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1669411840 bytes 2024-09-18T01:11:46.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1772863488 bytes 2024-09-18T01:11:46.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1800375296 bytes 2024-09-18T01:11:47.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1827326976 bytes 2024-09-18T01:11:47.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1836441600 bytes 2024-09-18T01:11:47.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1910034432 bytes 2024-09-18T01:11:47.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1977600000 bytes 2024-09-18T01:11:47.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2003960832 bytes 2024-09-18T01:11:47.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2055631872 bytes 2024-09-18T01:11:47.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2117809152 bytes 2024-09-18T01:11:48.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2164943872 bytes 2024-09-18T01:11:48.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2195890176 bytes 2024-09-18T01:11:48.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2235109376 bytes 2024-09-18T01:11:48.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2316804096 bytes 2024-09-18T01:11:48.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2352841728 bytes 2024-09-18T01:11:48.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2429581312 bytes 2024-09-18T01:11:48.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2454185984 bytes 2024-09-18T01:11:48.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2469076992 bytes 2024-09-18T01:11:49.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2569317376 bytes 2024-09-18T01:11:49.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2613955584 bytes 2024-09-18T01:11:49.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2625329152 bytes 2024-09-18T01:11:49.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2689767424 bytes 2024-09-18T01:11:49.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2733084672 bytes 2024-09-18T01:11:49.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2783332352 bytes 2024-09-18T01:11:49.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2803322880 bytes 2024-09-18T01:11:49.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2839720960 bytes 2024-09-18T01:11:50.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2858767360 bytes 2024-09-18T01:11:50.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2864941056 bytes 2024-09-18T01:11:50.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2871242752 bytes 2024-09-18T01:11:50.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2900101120 bytes 2024-09-18T01:11:50.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 2947666944 bytes 2024-09-18T01:11:50.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3024606208 bytes 2024-09-18T01:11:50.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3059966976 bytes 2024-09-18T01:11:50.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3060569088 bytes 2024-09-18T01:11:50.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3079038976 bytes 2024-09-18T01:11:51.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3111290880 bytes 2024-09-18T01:11:51.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3191983104 bytes 2024-09-18T01:11:51.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3233063936 bytes 2024-09-18T01:11:51.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3262280704 bytes 2024-09-18T01:11:51.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3360247808 bytes 2024-09-18T01:11:51.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3449284608 bytes 2024-09-18T01:11:51.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3484068864 bytes 2024-09-18T01:11:52.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3517873152 bytes 2024-09-18T01:11:52.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3526113280 bytes 2024-09-18T01:11:52.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3644368896 bytes 2024-09-18T01:11:52.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3717116928 bytes 2024-09-18T01:11:52.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3738698752 bytes 2024-09-18T01:11:52.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3774439424 bytes 2024-09-18T01:11:52.797 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3817619456 bytes 2024-09-18T01:11:52.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3880728576 bytes 2024-09-18T01:11:53.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3910665216 bytes 2024-09-18T01:11:53.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3956860928 bytes 2024-09-18T01:11:53.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 3974671360 bytes 2024-09-18T01:11:53.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 3982654464 bytes 2024-09-18T01:11:53.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4051495936 bytes 2024-09-18T01:11:53.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4102139904 bytes 2024-09-18T01:11:53.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4143325184 bytes 2024-09-18T01:11:53.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4187915264 bytes 2024-09-18T01:11:54.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4225352704 bytes 2024-09-18T01:11:54.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4325419008 bytes 2024-09-18T01:11:54.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4359749632 bytes 2024-09-18T01:11:54.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4399708160 bytes 2024-09-18T01:11:54.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4444333056 bytes 2024-09-18T01:11:54.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:11:54.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:11:54.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:11:54.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:11:54.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:11:54.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_15 2024-09-18T01:11:54.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 snapshot 128M 2024-09-18T01:11:54.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:54.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:54.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_15 128M 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:11:54.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:11:54.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:11:54.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:11:54.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_15 2024-09-18T01:11:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:11:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:11:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:11:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_15 snapshot 2024-09-18T01:11:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:54.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:11:54.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:11:54.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_15 snapshot 2024-09-18T01:11:55.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:11:55.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_15 2024-09-18T01:11:55.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_15': 2024-09-18T01:11:55.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:11:55.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12ad103ab2fd 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12ad103ab2fd 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:11:54 2024 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:11:54 2024 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:11:54 2024 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:11:55.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: b04593f0-b419-4bde-acad-9a17209e79b7 2024-09-18T01:11:55.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:11:55.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:11:55.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_15 2024-09-18T01:11:55.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:11:55.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:11:55.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:11:55.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:11:55.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:11:55.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:11:55.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:11:55.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 35935232 bytes 2024-09-18T01:11:55.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 158412800 bytes 2024-09-18T01:11:56.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 205813760 bytes 2024-09-18T01:11:56.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 301418496 bytes 2024-09-18T01:11:56.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 338407424 bytes 2024-09-18T01:11:56.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 394399744 bytes 2024-09-18T01:11:56.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 433930240 bytes 2024-09-18T01:11:56.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 454360064 bytes 2024-09-18T01:11:57.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 523030528 bytes 2024-09-18T01:11:57.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 581855232 bytes 2024-09-18T01:11:57.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 641912832 bytes 2024-09-18T01:11:57.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 647523328 bytes 2024-09-18T01:11:57.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 709156864 bytes 2024-09-18T01:11:57.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 725101568 bytes 2024-09-18T01:11:57.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 728071168 bytes 2024-09-18T01:11:57.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 766454784 bytes 2024-09-18T01:11:57.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 813504512 bytes 2024-09-18T01:11:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 835021824 bytes 2024-09-18T01:11:58.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 887699456 bytes 2024-09-18T01:11:58.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 943886336 bytes 2024-09-18T01:11:58.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1023118336 bytes 2024-09-18T01:11:58.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1061909504 bytes 2024-09-18T01:11:58.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1081606144 bytes 2024-09-18T01:11:58.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1141761024 bytes 2024-09-18T01:11:58.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1177647104 bytes 2024-09-18T01:11:59.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1249530880 bytes 2024-09-18T01:11:59.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1337456640 bytes 2024-09-18T01:11:59.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1430981632 bytes 2024-09-18T01:11:59.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1443228672 bytes 2024-09-18T01:11:59.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1449583616 bytes 2024-09-18T01:11:59.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1469941760 bytes 2024-09-18T01:11:59.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1530226688 bytes 2024-09-18T01:11:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1574283264 bytes 2024-09-18T01:12:00.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1613908992 bytes 2024-09-18T01:12:00.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1640761344 bytes 2024-09-18T01:12:00.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1701460992 bytes 2024-09-18T01:12:00.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1740619776 bytes 2024-09-18T01:12:00.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1759301632 bytes 2024-09-18T01:12:00.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1826793472 bytes 2024-09-18T01:12:00.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1864539136 bytes 2024-09-18T01:12:00.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1889394688 bytes 2024-09-18T01:12:01.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1949310976 bytes 2024-09-18T01:12:01.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1962257408 bytes 2024-09-18T01:12:01.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2053792768 bytes 2024-09-18T01:12:01.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2105472000 bytes 2024-09-18T01:12:01.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2153374720 bytes 2024-09-18T01:12:01.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2215185408 bytes 2024-09-18T01:12:01.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2235603968 bytes 2024-09-18T01:12:01.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2294254592 bytes 2024-09-18T01:12:02.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2308032512 bytes 2024-09-18T01:12:02.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2321207296 bytes 2024-09-18T01:12:02.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2323237888 bytes 2024-09-18T01:12:02.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2411579392 bytes 2024-09-18T01:12:02.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2414783488 bytes 2024-09-18T01:12:02.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2465660928 bytes 2024-09-18T01:12:02.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2498800640 bytes 2024-09-18T01:12:02.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2572666880 bytes 2024-09-18T01:12:03.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2600727552 bytes 2024-09-18T01:12:03.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2601333760 bytes 2024-09-18T01:12:03.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2633085952 bytes 2024-09-18T01:12:03.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2664311808 bytes 2024-09-18T01:12:03.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2735165440 bytes 2024-09-18T01:12:03.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2774263808 bytes 2024-09-18T01:12:03.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2812052480 bytes 2024-09-18T01:12:03.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2869856256 bytes 2024-09-18T01:12:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2870473728 bytes 2024-09-18T01:12:04.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2925542400 bytes 2024-09-18T01:12:04.151 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2926157824 bytes 2024-09-18T01:12:04.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3002920960 bytes 2024-09-18T01:12:04.458 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3068337152 bytes 2024-09-18T01:12:04.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3110709248 bytes 2024-09-18T01:12:04.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3184879616 bytes 2024-09-18T01:12:04.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3228402688 bytes 2024-09-18T01:12:04.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3261478912 bytes 2024-09-18T01:12:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3300590592 bytes 2024-09-18T01:12:05.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3339533312 bytes 2024-09-18T01:12:05.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3352381440 bytes 2024-09-18T01:12:05.443 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3406460928 bytes 2024-09-18T01:12:05.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3407096832 bytes 2024-09-18T01:12:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3462323200 bytes 2024-09-18T01:12:05.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3497911296 bytes 2024-09-18T01:12:05.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3556487168 bytes 2024-09-18T01:12:06.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3628917760 bytes 2024-09-18T01:12:06.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3682575360 bytes 2024-09-18T01:12:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3708540928 bytes 2024-09-18T01:12:06.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3709168640 bytes 2024-09-18T01:12:06.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3745416192 bytes 2024-09-18T01:12:06.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3788241920 bytes 2024-09-18T01:12:06.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3834008576 bytes 2024-09-18T01:12:06.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3868232704 bytes 2024-09-18T01:12:06.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3913443328 bytes 2024-09-18T01:12:07.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3927439360 bytes 2024-09-18T01:12:07.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3976790016 bytes 2024-09-18T01:12:07.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4022848512 bytes 2024-09-18T01:12:07.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4062436352 bytes 2024-09-18T01:12:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4085572608 bytes 2024-09-18T01:12:07.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4140531712 bytes 2024-09-18T01:12:07.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4161740800 bytes 2024-09-18T01:12:07.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4187212800 bytes 2024-09-18T01:12:08.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4240010240 bytes 2024-09-18T01:12:08.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4252607488 bytes 2024-09-18T01:12:08.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4329321472 bytes 2024-09-18T01:12:08.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4336339968 bytes 2024-09-18T01:12:08.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4432582656 bytes 2024-09-18T01:12:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4433205248 bytes 2024-09-18T01:12:08.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4456061952 bytes 2024-09-18T01:12:08.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4506775552 bytes 2024-09-18T01:12:08.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4507371520 bytes 2024-09-18T01:12:09.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4533206016 bytes 2024-09-18T01:12:09.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 4549202944 bytes 2024-09-18T01:12:09.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11100 IOs, 4566325248 bytes 2024-09-18T01:12:09.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11200 IOs, 4591214592 bytes 2024-09-18T01:12:09.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11300 IOs, 4595385344 bytes 2024-09-18T01:12:09.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11400 IOs, 4644811776 bytes 2024-09-18T01:12:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:12:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:12:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:12:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:12:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:12:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_16 2024-09-18T01:12:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 snapshot 128M 2024-09-18T01:12:09.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:12:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:12:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:12:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_16 128M 2024-09-18T01:12:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:12:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:12:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:12:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:12:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_16 2024-09-18T01:12:09.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:12:09.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:09.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:12:09.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:12:09.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_16 snapshot 2024-09-18T01:12:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:12:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:12:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_16 snapshot 2024-09-18T01:12:10.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:12:10.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_16 2024-09-18T01:12:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_16': 2024-09-18T01:12:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:12:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:12:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:12:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12b5ae2b7ef 2024-09-18T01:12:10.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12b5ae2b7ef 2024-09-18T01:12:10.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:12:10.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:12:10.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:12:10.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:12:10.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:12:09 2024 2024-09-18T01:12:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:12:09 2024 2024-09-18T01:12:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:12:09 2024 2024-09-18T01:12:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:12:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:12:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 1c96c904-8436-480e-934e-936299cf345d 2024-09-18T01:12:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:12:10.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:12:10.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_16 2024-09-18T01:12:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:12:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:12:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=12 2024-09-18T01:12:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:12:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:12:11.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 61684736 bytes 2024-09-18T01:12:11.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 84746240 bytes 2024-09-18T01:12:11.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 125146112 bytes 2024-09-18T01:12:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 169807872 bytes 2024-09-18T01:12:11.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 274920448 bytes 2024-09-18T01:12:11.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 306021376 bytes 2024-09-18T01:12:11.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 355340288 bytes 2024-09-18T01:12:11.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 387476480 bytes 2024-09-18T01:12:12.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 459766784 bytes 2024-09-18T01:12:12.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 535333888 bytes 2024-09-18T01:12:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 610503680 bytes 2024-09-18T01:12:12.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 611103744 bytes 2024-09-18T01:12:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 657173504 bytes 2024-09-18T01:12:12.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 672095232 bytes 2024-09-18T01:12:12.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 751912960 bytes 2024-09-18T01:12:12.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 802590720 bytes 2024-09-18T01:12:13.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 803772416 bytes 2024-09-18T01:12:13.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 837297152 bytes 2024-09-18T01:12:13.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 908291072 bytes 2024-09-18T01:12:13.426 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7592858571556192) > weight (0.25) 2024-09-18T01:12:13.427 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8497647126039322) > weight (0.25) 2024-09-18T01:12:13.427 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8636396704222615) > weight (0.25) 2024-09-18T01:12:13.427 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6651909770183714) > weight (0.25) 2024-09-18T01:12:13.427 INFO:tasks.rbd_mirror_thrash:waiting for 112 secs before thrashing 2024-09-18T01:12:13.443 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 956104704 bytes 2024-09-18T01:12:13.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1001371648 bytes 2024-09-18T01:12:13.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1032635392 bytes 2024-09-18T01:12:13.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1092458496 bytes 2024-09-18T01:12:13.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1125819392 bytes 2024-09-18T01:12:14.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1149414400 bytes 2024-09-18T01:12:14.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1178584064 bytes 2024-09-18T01:12:14.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1245312000 bytes 2024-09-18T01:12:14.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1311111168 bytes 2024-09-18T01:12:14.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1330541568 bytes 2024-09-18T01:12:14.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1411354624 bytes 2024-09-18T01:12:14.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1481904128 bytes 2024-09-18T01:12:14.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1482505216 bytes 2024-09-18T01:12:15.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1564236800 bytes 2024-09-18T01:12:15.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1596720128 bytes 2024-09-18T01:12:15.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1651152896 bytes 2024-09-18T01:12:15.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1684141056 bytes 2024-09-18T01:12:15.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1726280704 bytes 2024-09-18T01:12:15.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1771170816 bytes 2024-09-18T01:12:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1865485312 bytes 2024-09-18T01:12:16.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1938194432 bytes 2024-09-18T01:12:16.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1957054464 bytes 2024-09-18T01:12:16.272 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1992194048 bytes 2024-09-18T01:12:16.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2030097408 bytes 2024-09-18T01:12:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2075554816 bytes 2024-09-18T01:12:16.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2121767936 bytes 2024-09-18T01:12:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2161654784 bytes 2024-09-18T01:12:16.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2175141888 bytes 2024-09-18T01:12:16.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2213633024 bytes 2024-09-18T01:12:17.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2254829568 bytes 2024-09-18T01:12:17.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2307999744 bytes 2024-09-18T01:12:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2355019776 bytes 2024-09-18T01:12:17.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2405039104 bytes 2024-09-18T01:12:17.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2430027776 bytes 2024-09-18T01:12:17.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2486617088 bytes 2024-09-18T01:12:17.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2487258112 bytes 2024-09-18T01:12:17.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2499919872 bytes 2024-09-18T01:12:18.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2506178560 bytes 2024-09-18T01:12:18.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2552687616 bytes 2024-09-18T01:12:18.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2573100032 bytes 2024-09-18T01:12:18.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2600944640 bytes 2024-09-18T01:12:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2627510272 bytes 2024-09-18T01:12:18.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2680463360 bytes 2024-09-18T01:12:18.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2700229632 bytes 2024-09-18T01:12:18.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2721639424 bytes 2024-09-18T01:12:18.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2764214272 bytes 2024-09-18T01:12:19.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2803864576 bytes 2024-09-18T01:12:19.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2805204992 bytes 2024-09-18T01:12:19.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2832499712 bytes 2024-09-18T01:12:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2848671744 bytes 2024-09-18T01:12:19.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2889176064 bytes 2024-09-18T01:12:19.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2941577216 bytes 2024-09-18T01:12:19.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3020685312 bytes 2024-09-18T01:12:19.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3044999168 bytes 2024-09-18T01:12:20.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3072704512 bytes 2024-09-18T01:12:20.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3078868992 bytes 2024-09-18T01:12:20.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3111130112 bytes 2024-09-18T01:12:20.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3145204736 bytes 2024-09-18T01:12:20.443 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3174086656 bytes 2024-09-18T01:12:20.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3197839360 bytes 2024-09-18T01:12:20.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3266483200 bytes 2024-09-18T01:12:20.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3308067840 bytes 2024-09-18T01:12:20.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3328210944 bytes 2024-09-18T01:12:21.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3399744512 bytes 2024-09-18T01:12:21.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3452051456 bytes 2024-09-18T01:12:21.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3452652544 bytes 2024-09-18T01:12:21.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3497254912 bytes 2024-09-18T01:12:21.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3567710208 bytes 2024-09-18T01:12:21.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3618004992 bytes 2024-09-18T01:12:21.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3641614336 bytes 2024-09-18T01:12:21.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3734666240 bytes 2024-09-18T01:12:22.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3779452928 bytes 2024-09-18T01:12:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3819256832 bytes 2024-09-18T01:12:22.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3872571392 bytes 2024-09-18T01:12:22.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3878394880 bytes 2024-09-18T01:12:22.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 3937697792 bytes 2024-09-18T01:12:22.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:12:22.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:12:22.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:12:22.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:12:22.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:12:22.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_17 2024-09-18T01:12:22.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 snapshot 128M 2024-09-18T01:12:22.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:22.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_17 128M 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:22.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:12:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:12:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:12:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:12:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:22.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_17 2024-09-18T01:12:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:12:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:12:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:12:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_17 snapshot 2024-09-18T01:12:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:12:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:12:22.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_17 snapshot 2024-09-18T01:12:23.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:12:23.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_17 2024-09-18T01:12:23.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_17': 2024-09-18T01:12:23.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:12:23.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:12:23.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:12:23.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12bf31a0af2d 2024-09-18T01:12:23.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12bf31a0af2d 2024-09-18T01:12:23.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:12:23.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:12:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:12:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:12:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:12:22 2024 2024-09-18T01:12:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:12:22 2024 2024-09-18T01:12:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:12:22 2024 2024-09-18T01:12:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:12:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:12:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 3e9f111f-659a-4c25-a089-ca421c2ebd1a 2024-09-18T01:12:23.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:12:23.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:12:23.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_17 2024-09-18T01:12:23.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:23.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:23.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:12:23.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:12:23.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=7 2024-09-18T01:12:23.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:12:23.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:12:23.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 54422528 bytes 2024-09-18T01:12:23.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 94055424 bytes 2024-09-18T01:12:24.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 125750272 bytes 2024-09-18T01:12:24.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 212562944 bytes 2024-09-18T01:12:24.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 305488896 bytes 2024-09-18T01:12:24.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 335281152 bytes 2024-09-18T01:12:24.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 389901312 bytes 2024-09-18T01:12:24.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 437115904 bytes 2024-09-18T01:12:24.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 491219968 bytes 2024-09-18T01:12:25.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 563196928 bytes 2024-09-18T01:12:25.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 568628224 bytes 2024-09-18T01:12:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 592052224 bytes 2024-09-18T01:12:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 614341632 bytes 2024-09-18T01:12:25.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 627265536 bytes 2024-09-18T01:12:25.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 636635136 bytes 2024-09-18T01:12:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 656397312 bytes 2024-09-18T01:12:25.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 729338880 bytes 2024-09-18T01:12:25.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 734115840 bytes 2024-09-18T01:12:25.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 741454848 bytes 2024-09-18T01:12:26.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 774066176 bytes 2024-09-18T01:12:26.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 835154944 bytes 2024-09-18T01:12:26.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 848123904 bytes 2024-09-18T01:12:26.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 907441152 bytes 2024-09-18T01:12:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 910194688 bytes 2024-09-18T01:12:26.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 975287296 bytes 2024-09-18T01:12:26.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1044325376 bytes 2024-09-18T01:12:26.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1079058432 bytes 2024-09-18T01:12:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1112750080 bytes 2024-09-18T01:12:27.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1192543232 bytes 2024-09-18T01:12:27.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1235680256 bytes 2024-09-18T01:12:27.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1252543488 bytes 2024-09-18T01:12:27.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1313102848 bytes 2024-09-18T01:12:27.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1341489152 bytes 2024-09-18T01:12:27.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1402517504 bytes 2024-09-18T01:12:27.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1416334336 bytes 2024-09-18T01:12:28.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1491602432 bytes 2024-09-18T01:12:28.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1538461696 bytes 2024-09-18T01:12:28.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1555861504 bytes 2024-09-18T01:12:28.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1609414656 bytes 2024-09-18T01:12:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1761073152 bytes 2024-09-18T01:12:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1803840512 bytes 2024-09-18T01:12:29.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1834743808 bytes 2024-09-18T01:12:29.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1869635584 bytes 2024-09-18T01:12:29.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1926769664 bytes 2024-09-18T01:12:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1961681920 bytes 2024-09-18T01:12:29.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2024910848 bytes 2024-09-18T01:12:29.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2092384256 bytes 2024-09-18T01:12:29.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2131505152 bytes 2024-09-18T01:12:29.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2143978496 bytes 2024-09-18T01:12:29.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2144593920 bytes 2024-09-18T01:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2185840640 bytes 2024-09-18T01:12:30.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2265451520 bytes 2024-09-18T01:12:30.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2283676672 bytes 2024-09-18T01:12:30.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2336844800 bytes 2024-09-18T01:12:30.488 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9736926817559296) > weight (0.25) 2024-09-18T01:12:30.489 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-18T01:12:30.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-18T01:12:30.489 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-18T01:12:30.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:12:30.489+0000 7fef6f0b5640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 37165) UID: 1000 2024-09-18T01:12:30.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2356104192 bytes 2024-09-18T01:12:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:12:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:12:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:12:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:12:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:12:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_18 2024-09-18T01:12:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 snapshot 128M 2024-09-18T01:12:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:30.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:12:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:12:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:12:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_18 128M 2024-09-18T01:12:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:12:30.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:30.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:12:30.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:12:30.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:12:30.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:30.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_18 2024-09-18T01:12:30.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:12:30.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:30.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:12:30.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:12:30.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_18 snapshot 2024-09-18T01:12:30.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:30.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:30.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:12:30.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:12:30.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_18 snapshot 2024-09-18T01:12:31.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:12:31.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_18 2024-09-18T01:12:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_18': 2024-09-18T01:12:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:12:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:12:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:12:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12c8fe86e040 2024-09-18T01:12:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12c8fe86e040 2024-09-18T01:12:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:12:31.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:12:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:12:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:12:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:12:30 2024 2024-09-18T01:12:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:12:30 2024 2024-09-18T01:12:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:12:30 2024 2024-09-18T01:12:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:12:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:12:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 716a32d0-d4de-4f8b-8c5b-1431e7a8c859 2024-09-18T01:12:31.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:12:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:12:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_18 2024-09-18T01:12:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:12:31.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:12:31.658 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=13 2024-09-18T01:12:31.658 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:12:31.658 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:12:31.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 25820160 bytes 2024-09-18T01:12:31.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 102103040 bytes 2024-09-18T01:12:32.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 160237568 bytes 2024-09-18T01:12:32.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 215951360 bytes 2024-09-18T01:12:32.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 330087424 bytes 2024-09-18T01:12:32.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 390028288 bytes 2024-09-18T01:12:32.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 419739648 bytes 2024-09-18T01:12:33.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 468396032 bytes 2024-09-18T01:12:33.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 533580800 bytes 2024-09-18T01:12:33.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 560270336 bytes 2024-09-18T01:12:33.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 632344576 bytes 2024-09-18T01:12:33.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 661509120 bytes 2024-09-18T01:12:33.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 710315008 bytes 2024-09-18T01:12:33.797 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 725393408 bytes 2024-09-18T01:12:33.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 797924352 bytes 2024-09-18T01:12:34.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 869926912 bytes 2024-09-18T01:12:34.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 876968960 bytes 2024-09-18T01:12:34.490 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-18T01:12:34.490 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:12:34.490 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-18T01:12:34.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-18T01:12:34.491 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-18T01:12:34.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-18T01:12:34.494 INFO:tasks.rbd_mirror_thrash:waiting for 86 secs before thrashing 2024-09-18T01:12:34.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 966853632 bytes 2024-09-18T01:12:34.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 992906240 bytes 2024-09-18T01:12:34.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 1041437696 bytes 2024-09-18T01:12:34.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1075147776 bytes 2024-09-18T01:12:35.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1106094080 bytes 2024-09-18T01:12:35.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1142805504 bytes 2024-09-18T01:12:35.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1183912960 bytes 2024-09-18T01:12:35.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1217006592 bytes 2024-09-18T01:12:35.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1288454144 bytes 2024-09-18T01:12:35.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1289088000 bytes 2024-09-18T01:12:35.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1339481088 bytes 2024-09-18T01:12:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1366990848 bytes 2024-09-18T01:12:36.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1412334592 bytes 2024-09-18T01:12:36.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1464452096 bytes 2024-09-18T01:12:36.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1477567488 bytes 2024-09-18T01:12:36.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1479790592 bytes 2024-09-18T01:12:36.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1480434688 bytes 2024-09-18T01:12:36.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1515026432 bytes 2024-09-18T01:12:36.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1594364928 bytes 2024-09-18T01:12:36.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1614508032 bytes 2024-09-18T01:12:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1627659264 bytes 2024-09-18T01:12:37.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1671240704 bytes 2024-09-18T01:12:37.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1703009280 bytes 2024-09-18T01:12:37.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1785360384 bytes 2024-09-18T01:12:37.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1798232064 bytes 2024-09-18T01:12:37.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1872285696 bytes 2024-09-18T01:12:37.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1913921536 bytes 2024-09-18T01:12:37.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1981769728 bytes 2024-09-18T01:12:38.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2083175424 bytes 2024-09-18T01:12:38.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2093500416 bytes 2024-09-18T01:12:38.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2170856448 bytes 2024-09-18T01:12:38.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2188615680 bytes 2024-09-18T01:12:38.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2225566720 bytes 2024-09-18T01:12:38.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2278748160 bytes 2024-09-18T01:12:38.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2294519808 bytes 2024-09-18T01:12:38.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2383290368 bytes 2024-09-18T01:12:38.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2393549824 bytes 2024-09-18T01:12:39.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2430556160 bytes 2024-09-18T01:12:39.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2507590656 bytes 2024-09-18T01:12:39.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2539131904 bytes 2024-09-18T01:12:39.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2611895296 bytes 2024-09-18T01:12:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2676014080 bytes 2024-09-18T01:12:39.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2702576640 bytes 2024-09-18T01:12:39.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2783455232 bytes 2024-09-18T01:12:40.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2868934656 bytes 2024-09-18T01:12:40.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2873698304 bytes 2024-09-18T01:12:40.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2918131712 bytes 2024-09-18T01:12:40.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2957690880 bytes 2024-09-18T01:12:40.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 3013697536 bytes 2024-09-18T01:12:40.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3087516672 bytes 2024-09-18T01:12:40.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3123661824 bytes 2024-09-18T01:12:41.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3179595776 bytes 2024-09-18T01:12:41.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3256971264 bytes 2024-09-18T01:12:41.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3269238784 bytes 2024-09-18T01:12:41.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3310973952 bytes 2024-09-18T01:12:41.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3387510784 bytes 2024-09-18T01:12:41.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3421942784 bytes 2024-09-18T01:12:41.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3494932480 bytes 2024-09-18T01:12:41.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3514220544 bytes 2024-09-18T01:12:42.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3546433536 bytes 2024-09-18T01:12:42.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3579376640 bytes 2024-09-18T01:12:42.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3609759744 bytes 2024-09-18T01:12:42.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3673935872 bytes 2024-09-18T01:12:42.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3705218048 bytes 2024-09-18T01:12:42.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3765397504 bytes 2024-09-18T01:12:42.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3795650560 bytes 2024-09-18T01:12:42.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3834536960 bytes 2024-09-18T01:12:43.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3891958784 bytes 2024-09-18T01:12:43.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3916528640 bytes 2024-09-18T01:12:43.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3957145600 bytes 2024-09-18T01:12:43.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 4001838080 bytes 2024-09-18T01:12:43.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 4030397440 bytes 2024-09-18T01:12:43.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 4063963136 bytes 2024-09-18T01:12:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 4104093696 bytes 2024-09-18T01:12:43.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4110792704 bytes 2024-09-18T01:12:44.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4125059072 bytes 2024-09-18T01:12:44.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4129979392 bytes 2024-09-18T01:12:44.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4245494784 bytes 2024-09-18T01:12:44.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4286027776 bytes 2024-09-18T01:12:44.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4310972416 bytes 2024-09-18T01:12:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4318809088 bytes 2024-09-18T01:12:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_19 2024-09-18T01:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 snapshot 128M 2024-09-18T01:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_19 128M 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_19 2024-09-18T01:12:44.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:12:44.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:44.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:12:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:12:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_19 snapshot 2024-09-18T01:12:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:12:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:12:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_19 snapshot 2024-09-18T01:12:45.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:12:45.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_19 2024-09-18T01:12:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_19': 2024-09-18T01:12:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:12:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:12:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12defd422771 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12defd422771 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:12:44 2024 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:12:44 2024 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:12:44 2024 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 957171f2-b93c-457d-89be-5d45fdaad96c 2024-09-18T01:12:45.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:12:45.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:12:45.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_19 2024-09-18T01:12:45.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:45.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:45.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:12:45.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:12:45.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:12:45.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:12:45.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:12:45.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 41435136 bytes 2024-09-18T01:12:45.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 80622592 bytes 2024-09-18T01:12:46.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 90289152 bytes 2024-09-18T01:12:46.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 139382784 bytes 2024-09-18T01:12:46.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 156598272 bytes 2024-09-18T01:12:46.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 213387264 bytes 2024-09-18T01:12:46.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 259155968 bytes 2024-09-18T01:12:46.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 314204160 bytes 2024-09-18T01:12:46.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 320941056 bytes 2024-09-18T01:12:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 362711040 bytes 2024-09-18T01:12:47.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 416388096 bytes 2024-09-18T01:12:47.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 458744832 bytes 2024-09-18T01:12:47.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 530606080 bytes 2024-09-18T01:12:47.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 592271360 bytes 2024-09-18T01:12:47.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 630516736 bytes 2024-09-18T01:12:47.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 703917056 bytes 2024-09-18T01:12:47.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 736206848 bytes 2024-09-18T01:12:47.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 746079232 bytes 2024-09-18T01:12:48.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 825499648 bytes 2024-09-18T01:12:48.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 861718528 bytes 2024-09-18T01:12:48.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 893067264 bytes 2024-09-18T01:12:48.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 952985600 bytes 2024-09-18T01:12:48.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1021419520 bytes 2024-09-18T01:12:48.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1038459904 bytes 2024-09-18T01:12:48.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1065335808 bytes 2024-09-18T01:12:49.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1162393600 bytes 2024-09-18T01:12:49.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1206056960 bytes 2024-09-18T01:12:49.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1306809344 bytes 2024-09-18T01:12:49.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1357293568 bytes 2024-09-18T01:12:49.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1408884736 bytes 2024-09-18T01:12:49.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1455993856 bytes 2024-09-18T01:12:49.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1526760448 bytes 2024-09-18T01:12:49.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1549716480 bytes 2024-09-18T01:12:50.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1569456128 bytes 2024-09-18T01:12:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1594107904 bytes 2024-09-18T01:12:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1655207936 bytes 2024-09-18T01:12:50.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1691439104 bytes 2024-09-18T01:12:50.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1748707328 bytes 2024-09-18T01:12:50.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1768075264 bytes 2024-09-18T01:12:50.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1837440000 bytes 2024-09-18T01:12:50.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1838075904 bytes 2024-09-18T01:12:51.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1849424896 bytes 2024-09-18T01:12:51.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1897763840 bytes 2024-09-18T01:12:51.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1928797184 bytes 2024-09-18T01:12:51.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1934114816 bytes 2024-09-18T01:12:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1965663232 bytes 2024-09-18T01:12:51.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2029328384 bytes 2024-09-18T01:12:51.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2074133504 bytes 2024-09-18T01:12:51.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2113458176 bytes 2024-09-18T01:12:51.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2131004416 bytes 2024-09-18T01:12:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2208473088 bytes 2024-09-18T01:12:52.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2231597056 bytes 2024-09-18T01:12:52.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2243850240 bytes 2024-09-18T01:12:52.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2311451648 bytes 2024-09-18T01:12:52.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2345722880 bytes 2024-09-18T01:12:52.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2385588224 bytes 2024-09-18T01:12:52.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2419585024 bytes 2024-09-18T01:12:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2459552768 bytes 2024-09-18T01:12:53.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2500063232 bytes 2024-09-18T01:12:53.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2611222528 bytes 2024-09-18T01:12:53.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2673729536 bytes 2024-09-18T01:12:53.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2706894848 bytes 2024-09-18T01:12:53.658 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2723259392 bytes 2024-09-18T01:12:53.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2761879552 bytes 2024-09-18T01:12:53.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2772061184 bytes 2024-09-18T01:12:54.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2838076416 bytes 2024-09-18T01:12:54.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2875601920 bytes 2024-09-18T01:12:54.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2954721280 bytes 2024-09-18T01:12:54.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2989309952 bytes 2024-09-18T01:12:54.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3012676608 bytes 2024-09-18T01:12:54.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3066674176 bytes 2024-09-18T01:12:54.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3093599232 bytes 2024-09-18T01:12:54.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3117198336 bytes 2024-09-18T01:12:55.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3190393856 bytes 2024-09-18T01:12:55.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3244079104 bytes 2024-09-18T01:12:55.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3266422784 bytes 2024-09-18T01:12:55.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3318673408 bytes 2024-09-18T01:12:55.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3342057472 bytes 2024-09-18T01:12:55.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3393522688 bytes 2024-09-18T01:12:55.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3409890304 bytes 2024-09-18T01:12:55.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3420659712 bytes 2024-09-18T01:12:55.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3433896960 bytes 2024-09-18T01:12:56.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3459010560 bytes 2024-09-18T01:12:56.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3482598400 bytes 2024-09-18T01:12:56.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3544248320 bytes 2024-09-18T01:12:56.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3580104704 bytes 2024-09-18T01:12:56.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3638060032 bytes 2024-09-18T01:12:56.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3676005376 bytes 2024-09-18T01:12:56.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3721090048 bytes 2024-09-18T01:12:56.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3721708544 bytes 2024-09-18T01:12:57.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3759798272 bytes 2024-09-18T01:12:57.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3814556672 bytes 2024-09-18T01:12:57.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3889820672 bytes 2024-09-18T01:12:57.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3914487808 bytes 2024-09-18T01:12:57.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4001884160 bytes 2024-09-18T01:12:57.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4044083200 bytes 2024-09-18T01:12:57.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4071794688 bytes 2024-09-18T01:12:57.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4094501888 bytes 2024-09-18T01:12:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4095125504 bytes 2024-09-18T01:12:58.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4156244992 bytes 2024-09-18T01:12:58.338 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4177370112 bytes 2024-09-18T01:12:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4229962752 bytes 2024-09-18T01:12:58.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4251409408 bytes 2024-09-18T01:12:58.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4283016192 bytes 2024-09-18T01:12:58.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4351358976 bytes 2024-09-18T01:12:59.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4370131968 bytes 2024-09-18T01:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4464082944 bytes 2024-09-18T01:12:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4568050688 bytes 2024-09-18T01:12:59.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4590198784 bytes 2024-09-18T01:12:59.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 4622720000 bytes 2024-09-18T01:12:59.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:12:59.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:12:59.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:12:59.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:12:59.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:12:59.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_20 2024-09-18T01:12:59.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 snapshot 128M 2024-09-18T01:12:59.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:59.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:59.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:59.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:59.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:12:59.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:59.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:12:59.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:12:59.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:59.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_20 128M 2024-09-18T01:12:59.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:59.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:59.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:59.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:59.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:12:59.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:59.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:12:59.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:12:59.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:12:59.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:12:59.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_20 2024-09-18T01:12:59.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:12:59.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:59.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:12:59.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:12:59.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_20 snapshot 2024-09-18T01:12:59.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:12:59.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:12:59.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:12:59.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:12:59.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_20 snapshot 2024-09-18T01:13:00.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:13:00.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_20 2024-09-18T01:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_20': 2024-09-18T01:13:00.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12ea30c73c6e 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12ea30c73c6e 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:12:59 2024 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:12:59 2024 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:12:59 2024 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:13:00.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:13:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 820484e1-ad03-4296-9676-f12697dabdb5 2024-09-18T01:13:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:13:00.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:13:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_20 2024-09-18T01:13:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:13:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=8 2024-09-18T01:13:00.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:13:00.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:13:00.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 5969920 bytes 2024-09-18T01:13:00.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 55713792 bytes 2024-09-18T01:13:01.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 112212992 bytes 2024-09-18T01:13:01.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 146527232 bytes 2024-09-18T01:13:01.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 223817728 bytes 2024-09-18T01:13:01.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 264167424 bytes 2024-09-18T01:13:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 302021632 bytes 2024-09-18T01:13:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 322760704 bytes 2024-09-18T01:13:01.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 330140672 bytes 2024-09-18T01:13:01.910 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:13:01.908+0000 7f80d5d16640 -1 reset not still connected to 0x5641819d7c70 2024-09-18T01:13:01.910 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:13:01.908+0000 7f80d5d16640 -1 reset not still connected to 0x564188a38000 2024-09-18T01:13:01.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 370584576 bytes 2024-09-18T01:13:02.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 416266240 bytes 2024-09-18T01:13:02.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 453690368 bytes 2024-09-18T01:13:02.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 486678528 bytes 2024-09-18T01:13:02.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 553111552 bytes 2024-09-18T01:13:02.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 609893376 bytes 2024-09-18T01:13:02.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 645318656 bytes 2024-09-18T01:13:02.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 714862592 bytes 2024-09-18T01:13:03.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 805402624 bytes 2024-09-18T01:13:03.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 818071552 bytes 2024-09-18T01:13:03.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 852418560 bytes 2024-09-18T01:13:03.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 900085760 bytes 2024-09-18T01:13:03.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 941890560 bytes 2024-09-18T01:13:03.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1008445440 bytes 2024-09-18T01:13:03.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1063420928 bytes 2024-09-18T01:13:03.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1110626304 bytes 2024-09-18T01:13:04.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1137735680 bytes 2024-09-18T01:13:04.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1171872768 bytes 2024-09-18T01:13:04.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1221167104 bytes 2024-09-18T01:13:04.658 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1236930560 bytes 2024-09-18T01:13:04.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1237552128 bytes 2024-09-18T01:13:04.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1268314112 bytes 2024-09-18T01:13:04.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1279824896 bytes 2024-09-18T01:13:05.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1337555968 bytes 2024-09-18T01:13:05.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1395850240 bytes 2024-09-18T01:13:05.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1430091776 bytes 2024-09-18T01:13:05.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1514369024 bytes 2024-09-18T01:13:05.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1585010688 bytes 2024-09-18T01:13:05.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1606769664 bytes 2024-09-18T01:13:05.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1683710976 bytes 2024-09-18T01:13:06.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1737011200 bytes 2024-09-18T01:13:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1784923136 bytes 2024-09-18T01:13:06.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1822348288 bytes 2024-09-18T01:13:06.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1926674432 bytes 2024-09-18T01:13:06.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1959267328 bytes 2024-09-18T01:13:06.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2020847616 bytes 2024-09-18T01:13:06.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2108683264 bytes 2024-09-18T01:13:07.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2211416064 bytes 2024-09-18T01:13:07.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2245413888 bytes 2024-09-18T01:13:07.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2263896064 bytes 2024-09-18T01:13:07.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2324174848 bytes 2024-09-18T01:13:07.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2359697408 bytes 2024-09-18T01:13:07.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2440493056 bytes 2024-09-18T01:13:07.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2531064832 bytes 2024-09-18T01:13:08.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2606430208 bytes 2024-09-18T01:13:08.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2667958272 bytes 2024-09-18T01:13:08.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2707425280 bytes 2024-09-18T01:13:08.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2770828288 bytes 2024-09-18T01:13:08.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2826459136 bytes 2024-09-18T01:13:08.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:13:08.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:13:08.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:13:08.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:13:08.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:13:08.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_21 2024-09-18T01:13:08.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 snapshot 128M 2024-09-18T01:13:08.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:08.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:08.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_21 128M 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:13:08.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:08.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:13:08.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:13:08.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:13:08.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:08.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_21 2024-09-18T01:13:08.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:13:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:13:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:13:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_21 snapshot 2024-09-18T01:13:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:13:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:13:08.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_21 snapshot 2024-09-18T01:13:09.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:13:09.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_21 2024-09-18T01:13:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_21': 2024-09-18T01:13:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:13:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:13:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:13:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12f677bfd248 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12f677bfd248 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:13:08 2024 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:13:08 2024 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:13:08 2024 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 56bbe368-35dd-4230-b5bc-91e7ad099a39 2024-09-18T01:13:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:13:09.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:13:09.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_21 2024-09-18T01:13:09.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:09.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:09.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:13:09.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:13:09.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:13:09.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:13:09.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:13:09.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 49436672 bytes 2024-09-18T01:13:09.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 57081856 bytes 2024-09-18T01:13:10.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 97585152 bytes 2024-09-18T01:13:10.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 142679040 bytes 2024-09-18T01:13:10.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 210048000 bytes 2024-09-18T01:13:10.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 270807040 bytes 2024-09-18T01:13:10.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 335690752 bytes 2024-09-18T01:13:10.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 358534144 bytes 2024-09-18T01:13:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 444583936 bytes 2024-09-18T01:13:11.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 497331200 bytes 2024-09-18T01:13:11.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 556673024 bytes 2024-09-18T01:13:11.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 599434240 bytes 2024-09-18T01:13:11.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 679635968 bytes 2024-09-18T01:13:11.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 797208576 bytes 2024-09-18T01:13:11.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 852630528 bytes 2024-09-18T01:13:11.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 875937792 bytes 2024-09-18T01:13:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 907288576 bytes 2024-09-18T01:13:12.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 963908608 bytes 2024-09-18T01:13:12.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 1069191168 bytes 2024-09-18T01:13:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 1093422080 bytes 2024-09-18T01:13:12.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1101472768 bytes 2024-09-18T01:13:12.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1181002752 bytes 2024-09-18T01:13:12.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1265308672 bytes 2024-09-18T01:13:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1298162688 bytes 2024-09-18T01:13:13.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1338085376 bytes 2024-09-18T01:13:13.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1387157504 bytes 2024-09-18T01:13:13.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1467180032 bytes 2024-09-18T01:13:13.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1477498880 bytes 2024-09-18T01:13:13.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1527419904 bytes 2024-09-18T01:13:13.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1579465728 bytes 2024-09-18T01:13:14.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1643021312 bytes 2024-09-18T01:13:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1696907264 bytes 2024-09-18T01:13:14.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1726583808 bytes 2024-09-18T01:13:14.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1775494144 bytes 2024-09-18T01:13:14.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1815011328 bytes 2024-09-18T01:13:14.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1858664448 bytes 2024-09-18T01:13:14.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1909854208 bytes 2024-09-18T01:13:14.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1980561408 bytes 2024-09-18T01:13:15.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 2014739456 bytes 2024-09-18T01:13:15.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 2049347584 bytes 2024-09-18T01:13:15.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 2074575872 bytes 2024-09-18T01:13:15.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 2120857600 bytes 2024-09-18T01:13:15.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2136417280 bytes 2024-09-18T01:13:15.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2191630336 bytes 2024-09-18T01:13:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2216631296 bytes 2024-09-18T01:13:15.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2277324800 bytes 2024-09-18T01:13:16.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2296154112 bytes 2024-09-18T01:13:16.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2306564096 bytes 2024-09-18T01:13:16.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2356585472 bytes 2024-09-18T01:13:16.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2382044160 bytes 2024-09-18T01:13:16.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2427954176 bytes 2024-09-18T01:13:16.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2472815616 bytes 2024-09-18T01:13:16.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2500646912 bytes 2024-09-18T01:13:16.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2545259520 bytes 2024-09-18T01:13:17.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2589105152 bytes 2024-09-18T01:13:17.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2647894016 bytes 2024-09-18T01:13:17.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2696492032 bytes 2024-09-18T01:13:17.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2714179584 bytes 2024-09-18T01:13:17.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2770469888 bytes 2024-09-18T01:13:17.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2791547904 bytes 2024-09-18T01:13:17.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2821639168 bytes 2024-09-18T01:13:17.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2844234752 bytes 2024-09-18T01:13:17.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2888757248 bytes 2024-09-18T01:13:18.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2904116224 bytes 2024-09-18T01:13:18.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2940678144 bytes 2024-09-18T01:13:18.363 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 3018997760 bytes 2024-09-18T01:13:18.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3112433664 bytes 2024-09-18T01:13:18.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3180696576 bytes 2024-09-18T01:13:18.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3217509376 bytes 2024-09-18T01:13:18.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3234430976 bytes 2024-09-18T01:13:19.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3320116224 bytes 2024-09-18T01:13:19.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3375548416 bytes 2024-09-18T01:13:19.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3390532608 bytes 2024-09-18T01:13:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3508824064 bytes 2024-09-18T01:13:19.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3593506816 bytes 2024-09-18T01:13:19.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3617278976 bytes 2024-09-18T01:13:19.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3641730048 bytes 2024-09-18T01:13:20.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3675838464 bytes 2024-09-18T01:13:20.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3726554112 bytes 2024-09-18T01:13:20.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3778509824 bytes 2024-09-18T01:13:20.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3837254656 bytes 2024-09-18T01:13:20.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3899598848 bytes 2024-09-18T01:13:20.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3932656640 bytes 2024-09-18T01:13:20.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3939525632 bytes 2024-09-18T01:13:20.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3986048000 bytes 2024-09-18T01:13:21.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3988360192 bytes 2024-09-18T01:13:21.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 4013521920 bytes 2024-09-18T01:13:21.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 4094477312 bytes 2024-09-18T01:13:21.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 4127585280 bytes 2024-09-18T01:13:21.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 4128199680 bytes 2024-09-18T01:13:21.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 4188750848 bytes 2024-09-18T01:13:21.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4217351168 bytes 2024-09-18T01:13:21.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4261626880 bytes 2024-09-18T01:13:22.055 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4292227072 bytes 2024-09-18T01:13:22.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4412513280 bytes 2024-09-18T01:13:22.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4449359872 bytes 2024-09-18T01:13:22.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4499735552 bytes 2024-09-18T01:13:22.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4519571456 bytes 2024-09-18T01:13:22.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4524588032 bytes 2024-09-18T01:13:22.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4555937792 bytes 2024-09-18T01:13:22.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4640051200 bytes 2024-09-18T01:13:23.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4713715712 bytes 2024-09-18T01:13:23.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4834855936 bytes 2024-09-18T01:13:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4835470336 bytes 2024-09-18T01:13:23.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4850948096 bytes 2024-09-18T01:13:23.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4916454400 bytes 2024-09-18T01:13:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:13:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:13:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_22 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 snapshot 128M 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:13:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:13:23.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:23.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_22 128M 2024-09-18T01:13:23.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:13:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:13:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:13:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:13:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_22 2024-09-18T01:13:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:13:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:23.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:13:23.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:13:23.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_22 snapshot 2024-09-18T01:13:23.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:23.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:23.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:13:23.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:13:23.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_22 snapshot 2024-09-18T01:13:24.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:13:24.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_22 2024-09-18T01:13:24.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_22': 2024-09-18T01:13:24.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:13:24.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:13:24.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:13:24.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13022e12d308 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13022e12d308 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:13:23 2024 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:13:23 2024 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:13:23 2024 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 2b2d5ff2-a08d-4bda-9b74-e9d26e07305c 2024-09-18T01:13:24.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:13:24.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:13:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_22 2024-09-18T01:13:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:13:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:13:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=13 2024-09-18T01:13:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:13:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:13:24.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 63374336 bytes 2024-09-18T01:13:25.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 129803264 bytes 2024-09-18T01:13:25.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 161813504 bytes 2024-09-18T01:13:25.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 181858304 bytes 2024-09-18T01:13:25.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 299281408 bytes 2024-09-18T01:13:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 408094720 bytes 2024-09-18T01:13:25.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 450498560 bytes 2024-09-18T01:13:25.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 456983552 bytes 2024-09-18T01:13:26.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 516864000 bytes 2024-09-18T01:13:26.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 569804800 bytes 2024-09-18T01:13:26.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 649512960 bytes 2024-09-18T01:13:26.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 675930112 bytes 2024-09-18T01:13:26.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 725643264 bytes 2024-09-18T01:13:26.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 806876160 bytes 2024-09-18T01:13:26.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 828296192 bytes 2024-09-18T01:13:26.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 873262080 bytes 2024-09-18T01:13:27.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 911961088 bytes 2024-09-18T01:13:27.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 926414848 bytes 2024-09-18T01:13:27.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 975203328 bytes 2024-09-18T01:13:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 1040694272 bytes 2024-09-18T01:13:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1066798080 bytes 2024-09-18T01:13:27.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1067386880 bytes 2024-09-18T01:13:27.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1176239104 bytes 2024-09-18T01:13:27.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1181126656 bytes 2024-09-18T01:13:28.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1262520320 bytes 2024-09-18T01:13:28.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1283578880 bytes 2024-09-18T01:13:28.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1317314560 bytes 2024-09-18T01:13:28.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1328998400 bytes 2024-09-18T01:13:28.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1367783424 bytes 2024-09-18T01:13:28.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1391174656 bytes 2024-09-18T01:13:28.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1422907392 bytes 2024-09-18T01:13:28.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1484086272 bytes 2024-09-18T01:13:29.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1539399680 bytes 2024-09-18T01:13:29.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1613513728 bytes 2024-09-18T01:13:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1669548032 bytes 2024-09-18T01:13:29.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1696340992 bytes 2024-09-18T01:13:29.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1725481984 bytes 2024-09-18T01:13:29.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1759907840 bytes 2024-09-18T01:13:29.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1776918528 bytes 2024-09-18T01:13:29.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1834181632 bytes 2024-09-18T01:13:30.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1864347648 bytes 2024-09-18T01:13:30.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1881798656 bytes 2024-09-18T01:13:30.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1926842368 bytes 2024-09-18T01:13:30.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1966221312 bytes 2024-09-18T01:13:30.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2004756480 bytes 2024-09-18T01:13:30.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2078192640 bytes 2024-09-18T01:13:30.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2125572096 bytes 2024-09-18T01:13:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2158851072 bytes 2024-09-18T01:13:31.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2205319168 bytes 2024-09-18T01:13:31.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2294574080 bytes 2024-09-18T01:13:31.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2309950464 bytes 2024-09-18T01:13:31.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2338397184 bytes 2024-09-18T01:13:31.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2375721984 bytes 2024-09-18T01:13:31.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2448153600 bytes 2024-09-18T01:13:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2473440256 bytes 2024-09-18T01:13:32.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2509024256 bytes 2024-09-18T01:13:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2557888512 bytes 2024-09-18T01:13:32.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2578521088 bytes 2024-09-18T01:13:32.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2634703872 bytes 2024-09-18T01:13:32.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2647588864 bytes 2024-09-18T01:13:32.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2683293696 bytes 2024-09-18T01:13:32.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2726239232 bytes 2024-09-18T01:13:32.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2767480832 bytes 2024-09-18T01:13:33.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2806731776 bytes 2024-09-18T01:13:33.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2839622656 bytes 2024-09-18T01:13:33.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2882878464 bytes 2024-09-18T01:13:33.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2901194752 bytes 2024-09-18T01:13:33.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2937698304 bytes 2024-09-18T01:13:33.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2994240512 bytes 2024-09-18T01:13:33.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3074849792 bytes 2024-09-18T01:13:33.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3096452096 bytes 2024-09-18T01:13:34.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3126775808 bytes 2024-09-18T01:13:34.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3168476160 bytes 2024-09-18T01:13:34.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3177233408 bytes 2024-09-18T01:13:34.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3177832448 bytes 2024-09-18T01:13:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3197176832 bytes 2024-09-18T01:13:34.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3220900864 bytes 2024-09-18T01:13:34.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3281792000 bytes 2024-09-18T01:13:34.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3356968960 bytes 2024-09-18T01:13:35.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3405759488 bytes 2024-09-18T01:13:35.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3456509952 bytes 2024-09-18T01:13:35.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3486796800 bytes 2024-09-18T01:13:35.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3522488320 bytes 2024-09-18T01:13:35.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3600422912 bytes 2024-09-18T01:13:35.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3601032192 bytes 2024-09-18T01:13:35.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3625110528 bytes 2024-09-18T01:13:35.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3665218560 bytes 2024-09-18T01:13:36.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3704485888 bytes 2024-09-18T01:13:36.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3725129728 bytes 2024-09-18T01:13:36.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3773319168 bytes 2024-09-18T01:13:36.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3813061632 bytes 2024-09-18T01:13:36.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3828667392 bytes 2024-09-18T01:13:36.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3872353280 bytes 2024-09-18T01:13:36.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3948334080 bytes 2024-09-18T01:13:36.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4037469184 bytes 2024-09-18T01:13:37.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4069281792 bytes 2024-09-18T01:13:37.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4085251072 bytes 2024-09-18T01:13:37.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4139864064 bytes 2024-09-18T01:13:37.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4198516736 bytes 2024-09-18T01:13:37.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4241804288 bytes 2024-09-18T01:13:37.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4297909248 bytes 2024-09-18T01:13:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:13:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:13:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:13:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:13:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:13:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_23 2024-09-18T01:13:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 snapshot 128M 2024-09-18T01:13:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:37.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:13:37.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:37.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:13:37.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:13:37.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:37.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_23 128M 2024-09-18T01:13:37.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:13:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:13:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:13:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:13:37.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:37.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_23 2024-09-18T01:13:37.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:13:37.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:37.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:13:37.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:13:37.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_23 snapshot 2024-09-18T01:13:37.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:37.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:37.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:13:37.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:13:37.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_23 snapshot 2024-09-18T01:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:13:37.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_23 2024-09-18T01:13:37.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_23': 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 130e44cf0209 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.130e44cf0209 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:13:37 2024 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:13:37 2024 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:13:37 2024 2024-09-18T01:13:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:13:37.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:13:37.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: bf72281c-0895-415a-a775-377773847714 2024-09-18T01:13:37.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:13:37.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:13:37.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_23 2024-09-18T01:13:37.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:37.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:37.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:13:37.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:13:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=13 2024-09-18T01:13:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:13:37.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:13:38.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 78221312 bytes 2024-09-18T01:13:38.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 147422208 bytes 2024-09-18T01:13:38.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 148040704 bytes 2024-09-18T01:13:38.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 177685504 bytes 2024-09-18T01:13:38.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 213934080 bytes 2024-09-18T01:13:38.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 248241152 bytes 2024-09-18T01:13:38.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 312500224 bytes 2024-09-18T01:13:38.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 335721472 bytes 2024-09-18T01:13:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 379317248 bytes 2024-09-18T01:13:39.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 475278336 bytes 2024-09-18T01:13:39.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 518624256 bytes 2024-09-18T01:13:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 534002688 bytes 2024-09-18T01:13:39.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 576881664 bytes 2024-09-18T01:13:39.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 593329152 bytes 2024-09-18T01:13:39.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 598778880 bytes 2024-09-18T01:13:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 634412032 bytes 2024-09-18T01:13:40.087 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 672443392 bytes 2024-09-18T01:13:40.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 747165696 bytes 2024-09-18T01:13:40.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 747795456 bytes 2024-09-18T01:13:40.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 789231616 bytes 2024-09-18T01:13:40.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 816620544 bytes 2024-09-18T01:13:40.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 854255616 bytes 2024-09-18T01:13:40.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 872267776 bytes 2024-09-18T01:13:40.919 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 932270080 bytes 2024-09-18T01:13:41.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 958302208 bytes 2024-09-18T01:13:41.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1038117888 bytes 2024-09-18T01:13:41.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1121993728 bytes 2024-09-18T01:13:41.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1165796352 bytes 2024-09-18T01:13:41.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1186634752 bytes 2024-09-18T01:13:41.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1259753472 bytes 2024-09-18T01:13:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1272821760 bytes 2024-09-18T01:13:41.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1323960320 bytes 2024-09-18T01:13:42.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1346351104 bytes 2024-09-18T01:13:42.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1412616192 bytes 2024-09-18T01:13:42.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1468478464 bytes 2024-09-18T01:13:42.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1532217344 bytes 2024-09-18T01:13:42.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1563415552 bytes 2024-09-18T01:13:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1630605312 bytes 2024-09-18T01:13:42.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1671187456 bytes 2024-09-18T01:13:43.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1726449664 bytes 2024-09-18T01:13:43.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1776053248 bytes 2024-09-18T01:13:43.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1846164480 bytes 2024-09-18T01:13:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1934307328 bytes 2024-09-18T01:13:43.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1964818432 bytes 2024-09-18T01:13:43.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2004533248 bytes 2024-09-18T01:13:43.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2046266368 bytes 2024-09-18T01:13:44.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2155997184 bytes 2024-09-18T01:13:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2178864128 bytes 2024-09-18T01:13:44.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2192376832 bytes 2024-09-18T01:13:44.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2264947712 bytes 2024-09-18T01:13:44.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2295646208 bytes 2024-09-18T01:13:44.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2298331136 bytes 2024-09-18T01:13:44.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2334697472 bytes 2024-09-18T01:13:44.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2360107008 bytes 2024-09-18T01:13:45.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2415591424 bytes 2024-09-18T01:13:45.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2480453632 bytes 2024-09-18T01:13:45.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2546040832 bytes 2024-09-18T01:13:45.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2587124736 bytes 2024-09-18T01:13:45.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2605158400 bytes 2024-09-18T01:13:45.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2638909440 bytes 2024-09-18T01:13:45.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2710385664 bytes 2024-09-18T01:13:45.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2726811648 bytes 2024-09-18T01:13:46.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2758986752 bytes 2024-09-18T01:13:46.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2766193664 bytes 2024-09-18T01:13:46.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2785251328 bytes 2024-09-18T01:13:46.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2812525568 bytes 2024-09-18T01:13:46.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2830767104 bytes 2024-09-18T01:13:46.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2940146688 bytes 2024-09-18T01:13:46.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2964904960 bytes 2024-09-18T01:13:46.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3023122432 bytes 2024-09-18T01:13:47.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3085574144 bytes 2024-09-18T01:13:47.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3152005120 bytes 2024-09-18T01:13:47.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3195213824 bytes 2024-09-18T01:13:47.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3211700224 bytes 2024-09-18T01:13:47.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3269129216 bytes 2024-09-18T01:13:47.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3291612160 bytes 2024-09-18T01:13:47.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3329977344 bytes 2024-09-18T01:13:47.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3347929088 bytes 2024-09-18T01:13:47.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3404742656 bytes 2024-09-18T01:13:48.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3435419648 bytes 2024-09-18T01:13:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3441694720 bytes 2024-09-18T01:13:48.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3559081984 bytes 2024-09-18T01:13:48.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3570607104 bytes 2024-09-18T01:13:48.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3592802304 bytes 2024-09-18T01:13:48.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3637015552 bytes 2024-09-18T01:13:48.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3714871296 bytes 2024-09-18T01:13:48.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3732043776 bytes 2024-09-18T01:13:49.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3734448128 bytes 2024-09-18T01:13:49.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3787659264 bytes 2024-09-18T01:13:49.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3813172224 bytes 2024-09-18T01:13:49.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3902289920 bytes 2024-09-18T01:13:49.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3907311616 bytes 2024-09-18T01:13:49.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3927386112 bytes 2024-09-18T01:13:49.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3954904064 bytes 2024-09-18T01:13:49.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4044916736 bytes 2024-09-18T01:13:50.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4075274240 bytes 2024-09-18T01:13:50.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4101062656 bytes 2024-09-18T01:13:50.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4125710336 bytes 2024-09-18T01:13:50.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4153136128 bytes 2024-09-18T01:13:50.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4161052672 bytes 2024-09-18T01:13:50.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4223041536 bytes 2024-09-18T01:13:50.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4262111232 bytes 2024-09-18T01:13:50.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4308703232 bytes 2024-09-18T01:13:50.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:13:50.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:13:50.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:13:50.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:13:50.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:13:50.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_24 2024-09-18T01:13:50.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 snapshot 128M 2024-09-18T01:13:50.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:50.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:50.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:50.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:50.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:13:50.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_24 128M 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:13:50.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:13:50.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_24 2024-09-18T01:13:50.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:13:50.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:50.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:13:50.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:13:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_24 snapshot 2024-09-18T01:13:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:13:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:13:50.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_24 snapshot 2024-09-18T01:13:51.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:13:51.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_24 2024-09-18T01:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_24': 2024-09-18T01:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 131adb77025e 2024-09-18T01:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.131adb77025e 2024-09-18T01:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:13:51.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:13:50 2024 2024-09-18T01:13:51.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:13:50 2024 2024-09-18T01:13:51.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:13:50 2024 2024-09-18T01:13:51.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:13:51.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:13:51.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: e2f372b0-91b5-4e89-95d5-57ab346b50a4 2024-09-18T01:13:51.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:13:51.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:13:51.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_24 2024-09-18T01:13:51.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:13:51.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:13:51.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:13:51.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:13:51.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=12 2024-09-18T01:13:51.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:13:51.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:13:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 36532224 bytes 2024-09-18T01:13:52.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 53206016 bytes 2024-09-18T01:13:52.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 60444672 bytes 2024-09-18T01:13:52.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 149053440 bytes 2024-09-18T01:13:52.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 196174848 bytes 2024-09-18T01:13:52.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 259144704 bytes 2024-09-18T01:13:52.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 318144512 bytes 2024-09-18T01:13:53.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 347286528 bytes 2024-09-18T01:13:53.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 360402944 bytes 2024-09-18T01:13:53.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 410462208 bytes 2024-09-18T01:13:53.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 483616768 bytes 2024-09-18T01:13:53.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 516387840 bytes 2024-09-18T01:13:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 538909696 bytes 2024-09-18T01:13:53.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 610464768 bytes 2024-09-18T01:13:53.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 635797504 bytes 2024-09-18T01:13:53.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 647568384 bytes 2024-09-18T01:13:54.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 665644032 bytes 2024-09-18T01:13:54.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 724044800 bytes 2024-09-18T01:13:54.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 742280192 bytes 2024-09-18T01:13:54.406 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 742915072 bytes 2024-09-18T01:13:54.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 809926656 bytes 2024-09-18T01:13:54.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 907347968 bytes 2024-09-18T01:13:54.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 907957248 bytes 2024-09-18T01:13:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 946414592 bytes 2024-09-18T01:13:55.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1038306304 bytes 2024-09-18T01:13:55.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1082380288 bytes 2024-09-18T01:13:55.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1140944896 bytes 2024-09-18T01:13:55.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1141591040 bytes 2024-09-18T01:13:55.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1193758720 bytes 2024-09-18T01:13:55.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1194396672 bytes 2024-09-18T01:13:55.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1267917824 bytes 2024-09-18T01:13:55.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1316849664 bytes 2024-09-18T01:13:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1385535488 bytes 2024-09-18T01:13:56.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1448420352 bytes 2024-09-18T01:13:56.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1538830336 bytes 2024-09-18T01:13:56.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1559446528 bytes 2024-09-18T01:13:56.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1614090240 bytes 2024-09-18T01:13:56.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1614723072 bytes 2024-09-18T01:13:56.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1687427072 bytes 2024-09-18T01:13:57.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1780789248 bytes 2024-09-18T01:13:57.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1811764224 bytes 2024-09-18T01:13:57.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1837188096 bytes 2024-09-18T01:13:57.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1853786112 bytes 2024-09-18T01:13:57.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1870927872 bytes 2024-09-18T01:13:57.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1903241216 bytes 2024-09-18T01:13:57.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1926684672 bytes 2024-09-18T01:13:57.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1977643008 bytes 2024-09-18T01:13:58.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2043766784 bytes 2024-09-18T01:13:58.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2078700544 bytes 2024-09-18T01:13:58.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2110155776 bytes 2024-09-18T01:13:58.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2169170944 bytes 2024-09-18T01:13:58.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2228936704 bytes 2024-09-18T01:13:58.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2294915072 bytes 2024-09-18T01:13:58.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2327607296 bytes 2024-09-18T01:13:58.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2374474752 bytes 2024-09-18T01:13:59.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2385559552 bytes 2024-09-18T01:13:59.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2424698880 bytes 2024-09-18T01:13:59.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2506394624 bytes 2024-09-18T01:13:59.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2520112128 bytes 2024-09-18T01:13:59.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2542488576 bytes 2024-09-18T01:13:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2574311424 bytes 2024-09-18T01:13:59.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2593820672 bytes 2024-09-18T01:13:59.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2642649088 bytes 2024-09-18T01:14:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2660531200 bytes 2024-09-18T01:14:00.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2764896256 bytes 2024-09-18T01:14:00.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2824156160 bytes 2024-09-18T01:14:00.495 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.29750238206387525) > weight (0.25) 2024-09-18T01:14:00.495 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7526257550467886) > weight (0.25) 2024-09-18T01:14:00.495 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5151830436257727) > weight (0.25) 2024-09-18T01:14:00.495 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2024-09-18T01:14:00.495 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2024-09-18T01:14:00.496 INFO:tasks.rbd_mirror_thrash:waiting for 3 secs before reviving daemons 2024-09-18T01:14:00.497 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:14:00.495+0000 7f285808b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 37231) UID: 1000 2024-09-18T01:14:00.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2883526656 bytes 2024-09-18T01:14:00.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2931932160 bytes 2024-09-18T01:14:00.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2985225216 bytes 2024-09-18T01:14:00.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3015941120 bytes 2024-09-18T01:14:01.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3102937088 bytes 2024-09-18T01:14:01.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3154731008 bytes 2024-09-18T01:14:01.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3215154176 bytes 2024-09-18T01:14:01.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3259750400 bytes 2024-09-18T01:14:01.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3274552320 bytes 2024-09-18T01:14:01.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3302497280 bytes 2024-09-18T01:14:01.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3361976320 bytes 2024-09-18T01:14:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3400713216 bytes 2024-09-18T01:14:02.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3425781760 bytes 2024-09-18T01:14:02.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3440552960 bytes 2024-09-18T01:14:02.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3502360576 bytes 2024-09-18T01:14:02.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3547998208 bytes 2024-09-18T01:14:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3600888832 bytes 2024-09-18T01:14:02.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3658616832 bytes 2024-09-18T01:14:02.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3687510016 bytes 2024-09-18T01:14:02.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3688114176 bytes 2024-09-18T01:14:03.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3705065472 bytes 2024-09-18T01:14:03.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3757019136 bytes 2024-09-18T01:14:03.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3828177920 bytes 2024-09-18T01:14:03.497 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2024-09-18T01:14:03.497 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:14:03.497 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2024-09-18T01:14:03.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2024-09-18T01:14:03.498 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2024-09-18T01:14:03.502 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2024-09-18T01:14:03.502 INFO:tasks.rbd_mirror_thrash:waiting for 92 secs before thrashing 2024-09-18T01:14:03.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3951377408 bytes 2024-09-18T01:14:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3974417408 bytes 2024-09-18T01:14:03.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4033509376 bytes 2024-09-18T01:14:03.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_25 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 snapshot 128M 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:14:03.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_25 128M 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:14:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:14:03.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:14:03.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:03.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_25 2024-09-18T01:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_25 snapshot 2024-09-18T01:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:04.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:14:04.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:04.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_25 snapshot 2024-09-18T01:14:04.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:14:04.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_25 2024-09-18T01:14:04.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_25': 2024-09-18T01:14:04.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:14:04.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:14:04.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:14:04.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 132d5a95ca3f 2024-09-18T01:14:04.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.132d5a95ca3f 2024-09-18T01:14:04.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:14:04.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:14:04.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:14:04.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:14:04.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:04 2024 2024-09-18T01:14:04.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:04 2024 2024-09-18T01:14:04.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:04 2024 2024-09-18T01:14:04.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:14:04.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:14:04.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 87947b3d-24d4-4600-a788-3246abc4083b 2024-09-18T01:14:04.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:14:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:14:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_25 2024-09-18T01:14:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:14:04.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:14:04.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=6 2024-09-18T01:14:04.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:14:04.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:14:05.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 80935936 bytes 2024-09-18T01:14:05.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 162765824 bytes 2024-09-18T01:14:05.428 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-18T01:14:05.428 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-18T01:14:05.428 INFO:tasks.rbd_mirror_thrash:waiting for 3 secs before reviving daemons 2024-09-18T01:14:05.430 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:14:05.428+0000 7f3eb871f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 40287) UID: 1000 2024-09-18T01:14:05.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 214575104 bytes 2024-09-18T01:14:05.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 225736704 bytes 2024-09-18T01:14:05.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 257788928 bytes 2024-09-18T01:14:05.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 324328448 bytes 2024-09-18T01:14:06.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 372319232 bytes 2024-09-18T01:14:06.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 380463104 bytes 2024-09-18T01:14:06.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 405190656 bytes 2024-09-18T01:14:06.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 434780160 bytes 2024-09-18T01:14:06.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 490724352 bytes 2024-09-18T01:14:06.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 511326208 bytes 2024-09-18T01:14:06.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 568346624 bytes 2024-09-18T01:14:06.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 627907584 bytes 2024-09-18T01:14:07.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 710462464 bytes 2024-09-18T01:14:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 736233472 bytes 2024-09-18T01:14:07.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 756489216 bytes 2024-09-18T01:14:07.391 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 782151680 bytes 2024-09-18T01:14:07.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 811671552 bytes 2024-09-18T01:14:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 852496384 bytes 2024-09-18T01:14:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 935193600 bytes 2024-09-18T01:14:07.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 979390464 bytes 2024-09-18T01:14:08.055 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1012530176 bytes 2024-09-18T01:14:08.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1049761792 bytes 2024-09-18T01:14:08.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1074891776 bytes 2024-09-18T01:14:08.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1075499008 bytes 2024-09-18T01:14:08.429 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-18T01:14:08.429 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:14:08.430 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-18T01:14:08.430 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-18T01:14:08.430 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-18T01:14:08.433 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-18T01:14:08.433 INFO:tasks.rbd_mirror_thrash:waiting for 107 secs before thrashing 2024-09-18T01:14:08.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1093320704 bytes 2024-09-18T01:14:08.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1181080576 bytes 2024-09-18T01:14:08.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1223922688 bytes 2024-09-18T01:14:08.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1240120320 bytes 2024-09-18T01:14:09.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1241920512 bytes 2024-09-18T01:14:09.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1292372992 bytes 2024-09-18T01:14:09.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1360596992 bytes 2024-09-18T01:14:09.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1365587968 bytes 2024-09-18T01:14:09.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1446159360 bytes 2024-09-18T01:14:09.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1534681088 bytes 2024-09-18T01:14:09.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1538151424 bytes 2024-09-18T01:14:09.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1568452608 bytes 2024-09-18T01:14:10.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1625202688 bytes 2024-09-18T01:14:10.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1639156736 bytes 2024-09-18T01:14:10.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1668558848 bytes 2024-09-18T01:14:10.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1741127680 bytes 2024-09-18T01:14:10.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1805815808 bytes 2024-09-18T01:14:10.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1806435328 bytes 2024-09-18T01:14:10.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1845352448 bytes 2024-09-18T01:14:10.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1904186368 bytes 2024-09-18T01:14:11.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:14:11.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:14:11.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:14:11.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:11.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_26 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 snapshot 128M 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_26 128M 2024-09-18T01:14:11.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:11.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:11.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:11.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:11.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:14:11.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:11.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:14:11.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:14:11.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:14:11.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:11.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_26 2024-09-18T01:14:11.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:14:11.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:11.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:14:11.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:11.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_26 snapshot 2024-09-18T01:14:11.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:11.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:11.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:14:11.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:11.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_26 snapshot 2024-09-18T01:14:11.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:14:11.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_26 2024-09-18T01:14:11.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_26': 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 133fcd8a3d7a 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.133fcd8a3d7a 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:11 2024 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:11 2024 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:11 2024 2024-09-18T01:14:11.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:14:11.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:14:11.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: ca67ff60-0889-4b07-bb52-b0e5f8ac56c7 2024-09-18T01:14:11.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:14:11.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:14:11.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_26 2024-09-18T01:14:11.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:11.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:11.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:14:11.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:14:11.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=10 2024-09-18T01:14:11.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:14:11.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:14:11.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 67605504 bytes 2024-09-18T01:14:11.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 172965888 bytes 2024-09-18T01:14:11.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 193143808 bytes 2024-09-18T01:14:12.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 211194880 bytes 2024-09-18T01:14:12.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 241390592 bytes 2024-09-18T01:14:12.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 259596288 bytes 2024-09-18T01:14:12.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 300562432 bytes 2024-09-18T01:14:12.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 388078592 bytes 2024-09-18T01:14:12.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 420074496 bytes 2024-09-18T01:14:12.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 429523968 bytes 2024-09-18T01:14:12.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 485665792 bytes 2024-09-18T01:14:13.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 515761152 bytes 2024-09-18T01:14:13.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 537369600 bytes 2024-09-18T01:14:13.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 576711680 bytes 2024-09-18T01:14:13.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 593449984 bytes 2024-09-18T01:14:13.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 594060288 bytes 2024-09-18T01:14:13.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 646100992 bytes 2024-09-18T01:14:13.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 679727104 bytes 2024-09-18T01:14:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 731740160 bytes 2024-09-18T01:14:14.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 815254528 bytes 2024-09-18T01:14:14.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 902737920 bytes 2024-09-18T01:14:14.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 934233088 bytes 2024-09-18T01:14:14.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 985300992 bytes 2024-09-18T01:14:14.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1049699328 bytes 2024-09-18T01:14:14.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1082484736 bytes 2024-09-18T01:14:14.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1099671552 bytes 2024-09-18T01:14:14.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1124627456 bytes 2024-09-18T01:14:15.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1202279424 bytes 2024-09-18T01:14:15.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1229243392 bytes 2024-09-18T01:14:15.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1266308096 bytes 2024-09-18T01:14:15.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1276738560 bytes 2024-09-18T01:14:15.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1339500544 bytes 2024-09-18T01:14:15.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1351902208 bytes 2024-09-18T01:14:15.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1381020672 bytes 2024-09-18T01:14:15.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1390727168 bytes 2024-09-18T01:14:16.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1467235328 bytes 2024-09-18T01:14:16.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1518377984 bytes 2024-09-18T01:14:16.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1574962176 bytes 2024-09-18T01:14:16.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1628321792 bytes 2024-09-18T01:14:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1648594944 bytes 2024-09-18T01:14:16.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1724706816 bytes 2024-09-18T01:14:16.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1725307904 bytes 2024-09-18T01:14:17.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1783309312 bytes 2024-09-18T01:14:17.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1804346368 bytes 2024-09-18T01:14:17.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1840722944 bytes 2024-09-18T01:14:17.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1906014208 bytes 2024-09-18T01:14:17.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1958988800 bytes 2024-09-18T01:14:17.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2035744768 bytes 2024-09-18T01:14:17.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2047935488 bytes 2024-09-18T01:14:18.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2141272064 bytes 2024-09-18T01:14:18.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2188875776 bytes 2024-09-18T01:14:18.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2225841152 bytes 2024-09-18T01:14:18.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2228191232 bytes 2024-09-18T01:14:18.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2237710336 bytes 2024-09-18T01:14:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2273303552 bytes 2024-09-18T01:14:18.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2307049472 bytes 2024-09-18T01:14:18.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2309990400 bytes 2024-09-18T01:14:18.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2331321344 bytes 2024-09-18T01:14:19.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2352766976 bytes 2024-09-18T01:14:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2405265408 bytes 2024-09-18T01:14:19.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2452860928 bytes 2024-09-18T01:14:19.458 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2495860736 bytes 2024-09-18T01:14:19.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2518675456 bytes 2024-09-18T01:14:19.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2553533440 bytes 2024-09-18T01:14:19.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2565510144 bytes 2024-09-18T01:14:19.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2627938304 bytes 2024-09-18T01:14:20.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2666425344 bytes 2024-09-18T01:14:20.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2667040768 bytes 2024-09-18T01:14:20.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2699466752 bytes 2024-09-18T01:14:20.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2723170304 bytes 2024-09-18T01:14:20.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2780562432 bytes 2024-09-18T01:14:20.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2876909568 bytes 2024-09-18T01:14:20.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 2951590912 bytes 2024-09-18T01:14:20.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 2992934912 bytes 2024-09-18T01:14:21.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3046831104 bytes 2024-09-18T01:14:21.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3085302784 bytes 2024-09-18T01:14:21.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3104557056 bytes 2024-09-18T01:14:21.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:14:21.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:14:21.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:14:21.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_27 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 snapshot 128M 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:21.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_27 128M 2024-09-18T01:14:21.443 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:21.443 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:21.443 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:21.443 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:21.443 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:14:21.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:21.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:14:21.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:14:21.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:14:21.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:21.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_27 2024-09-18T01:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_27 snapshot 2024-09-18T01:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:21.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_27 snapshot 2024-09-18T01:14:22.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:14:22.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_27 2024-09-18T01:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_27': 2024-09-18T01:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 134bde49eed4 2024-09-18T01:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.134bde49eed4 2024-09-18T01:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:14:22.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:14:22.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:21 2024 2024-09-18T01:14:22.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:21 2024 2024-09-18T01:14:22.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:21 2024 2024-09-18T01:14:22.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:14:22.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:14:22.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: d521f001-ba66-4693-9d40-d7a054a004d9 2024-09-18T01:14:22.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:14:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:14:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_27 2024-09-18T01:14:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:14:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:14:22.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=9 2024-09-18T01:14:22.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:14:22.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:14:22.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 61550592 bytes 2024-09-18T01:14:22.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 107048960 bytes 2024-09-18T01:14:22.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 125268992 bytes 2024-09-18T01:14:23.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 192107520 bytes 2024-09-18T01:14:23.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 270116864 bytes 2024-09-18T01:14:23.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 361372672 bytes 2024-09-18T01:14:23.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 419821568 bytes 2024-09-18T01:14:23.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 472483840 bytes 2024-09-18T01:14:23.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 525867008 bytes 2024-09-18T01:14:23.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 538635264 bytes 2024-09-18T01:14:23.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 562824192 bytes 2024-09-18T01:14:24.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 563437568 bytes 2024-09-18T01:14:24.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 636114944 bytes 2024-09-18T01:14:24.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 653089792 bytes 2024-09-18T01:14:24.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 671474688 bytes 2024-09-18T01:14:24.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 689146880 bytes 2024-09-18T01:14:24.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 712835072 bytes 2024-09-18T01:14:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 779551744 bytes 2024-09-18T01:14:24.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 797266944 bytes 2024-09-18T01:14:25.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 874118144 bytes 2024-09-18T01:14:25.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 939944960 bytes 2024-09-18T01:14:25.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1009813504 bytes 2024-09-18T01:14:25.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1038445568 bytes 2024-09-18T01:14:25.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1058015232 bytes 2024-09-18T01:14:25.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1093841920 bytes 2024-09-18T01:14:25.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1125502976 bytes 2024-09-18T01:14:25.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1169271808 bytes 2024-09-18T01:14:26.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1262368768 bytes 2024-09-18T01:14:26.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1321942016 bytes 2024-09-18T01:14:26.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1368739840 bytes 2024-09-18T01:14:26.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1386934272 bytes 2024-09-18T01:14:26.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1420268544 bytes 2024-09-18T01:14:26.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1461428224 bytes 2024-09-18T01:14:26.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1500573696 bytes 2024-09-18T01:14:27.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1564301312 bytes 2024-09-18T01:14:27.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1625114624 bytes 2024-09-18T01:14:27.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1668126720 bytes 2024-09-18T01:14:27.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1701871616 bytes 2024-09-18T01:14:27.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1759057920 bytes 2024-09-18T01:14:27.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1810186240 bytes 2024-09-18T01:14:27.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1876360192 bytes 2024-09-18T01:14:27.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1901998080 bytes 2024-09-18T01:14:28.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1993412608 bytes 2024-09-18T01:14:28.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2016640000 bytes 2024-09-18T01:14:28.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2065760256 bytes 2024-09-18T01:14:28.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2077750272 bytes 2024-09-18T01:14:28.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2101884928 bytes 2024-09-18T01:14:28.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2110275584 bytes 2024-09-18T01:14:28.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2157292544 bytes 2024-09-18T01:14:29.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2254898176 bytes 2024-09-18T01:14:29.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2271502336 bytes 2024-09-18T01:14:29.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2336465920 bytes 2024-09-18T01:14:29.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2452498432 bytes 2024-09-18T01:14:29.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2516142080 bytes 2024-09-18T01:14:29.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2553549824 bytes 2024-09-18T01:14:29.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2591570944 bytes 2024-09-18T01:14:29.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2632000512 bytes 2024-09-18T01:14:30.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2656147456 bytes 2024-09-18T01:14:30.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2720679936 bytes 2024-09-18T01:14:30.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2756521984 bytes 2024-09-18T01:14:30.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2839478272 bytes 2024-09-18T01:14:30.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2905811968 bytes 2024-09-18T01:14:30.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2960914432 bytes 2024-09-18T01:14:30.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2994743296 bytes 2024-09-18T01:14:31.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 3057815552 bytes 2024-09-18T01:14:31.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 3099328512 bytes 2024-09-18T01:14:31.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3134786560 bytes 2024-09-18T01:14:31.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3135696896 bytes 2024-09-18T01:14:31.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_28 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 snapshot 128M 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:31.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:31.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:14:31.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:31.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_28 128M 2024-09-18T01:14:31.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:31.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:31.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:31.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:31.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:14:31.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:31.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:14:31.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:14:31.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:14:31.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:31.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_28 2024-09-18T01:14:31.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:14:31.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:31.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:14:31.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:31.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_28 snapshot 2024-09-18T01:14:31.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:31.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:31.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:14:31.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:31.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_28 snapshot 2024-09-18T01:14:32.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:14:32.423 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_28 2024-09-18T01:14:32.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_28': 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13576e1c7f0a 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13576e1c7f0a 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:31 2024 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:31 2024 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:31 2024 2024-09-18T01:14:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:14:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:14:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 24274cb0-c250-4f16-a3b6-16755405ece7 2024-09-18T01:14:32.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:14:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:14:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_28 2024-09-18T01:14:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:14:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:14:32.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=13 2024-09-18T01:14:32.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:14:32.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:14:32.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 8785920 bytes 2024-09-18T01:14:32.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 26225664 bytes 2024-09-18T01:14:32.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 103701504 bytes 2024-09-18T01:14:32.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 146134016 bytes 2024-09-18T01:14:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 164043776 bytes 2024-09-18T01:14:33.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 237166592 bytes 2024-09-18T01:14:33.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 299536384 bytes 2024-09-18T01:14:33.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 384259072 bytes 2024-09-18T01:14:33.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 416150528 bytes 2024-09-18T01:14:33.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 484938752 bytes 2024-09-18T01:14:33.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 507334656 bytes 2024-09-18T01:14:34.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 553791488 bytes 2024-09-18T01:14:34.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 579090432 bytes 2024-09-18T01:14:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 601413632 bytes 2024-09-18T01:14:34.458 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 647834624 bytes 2024-09-18T01:14:34.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 692584448 bytes 2024-09-18T01:14:34.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 713952256 bytes 2024-09-18T01:14:34.815 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 734635008 bytes 2024-09-18T01:14:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 770670592 bytes 2024-09-18T01:14:35.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 791208960 bytes 2024-09-18T01:14:35.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 834332672 bytes 2024-09-18T01:14:35.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 859555840 bytes 2024-09-18T01:14:35.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 860196864 bytes 2024-09-18T01:14:35.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 900667392 bytes 2024-09-18T01:14:35.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 922778624 bytes 2024-09-18T01:14:35.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 945696768 bytes 2024-09-18T01:14:35.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 966126592 bytes 2024-09-18T01:14:35.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1005864960 bytes 2024-09-18T01:14:36.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1029475328 bytes 2024-09-18T01:14:36.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1094751232 bytes 2024-09-18T01:14:36.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1146189824 bytes 2024-09-18T01:14:36.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1207966720 bytes 2024-09-18T01:14:36.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1225376768 bytes 2024-09-18T01:14:36.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1233990656 bytes 2024-09-18T01:14:36.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1238816768 bytes 2024-09-18T01:14:36.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1287047168 bytes 2024-09-18T01:14:37.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1328304128 bytes 2024-09-18T01:14:37.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1355095040 bytes 2024-09-18T01:14:37.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1401495552 bytes 2024-09-18T01:14:37.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1426745344 bytes 2024-09-18T01:14:37.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1466698752 bytes 2024-09-18T01:14:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1520841728 bytes 2024-09-18T01:14:37.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1536923648 bytes 2024-09-18T01:14:37.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1571045376 bytes 2024-09-18T01:14:38.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1684296704 bytes 2024-09-18T01:14:38.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1746288640 bytes 2024-09-18T01:14:38.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1777538048 bytes 2024-09-18T01:14:38.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1856180224 bytes 2024-09-18T01:14:38.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1891880960 bytes 2024-09-18T01:14:38.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1946217472 bytes 2024-09-18T01:14:38.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2002536448 bytes 2024-09-18T01:14:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2056630272 bytes 2024-09-18T01:14:39.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2106683392 bytes 2024-09-18T01:14:39.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2162089984 bytes 2024-09-18T01:14:39.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2245595136 bytes 2024-09-18T01:14:39.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2308825088 bytes 2024-09-18T01:14:39.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2333069312 bytes 2024-09-18T01:14:39.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2352855040 bytes 2024-09-18T01:14:40.055 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2374224896 bytes 2024-09-18T01:14:40.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2402702336 bytes 2024-09-18T01:14:40.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2477971456 bytes 2024-09-18T01:14:40.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2537823232 bytes 2024-09-18T01:14:40.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2572551168 bytes 2024-09-18T01:14:40.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2623947776 bytes 2024-09-18T01:14:40.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2648145920 bytes 2024-09-18T01:14:40.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2670624768 bytes 2024-09-18T01:14:41.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2733337600 bytes 2024-09-18T01:14:41.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2799444992 bytes 2024-09-18T01:14:41.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2854804480 bytes 2024-09-18T01:14:41.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2899589120 bytes 2024-09-18T01:14:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2964910080 bytes 2024-09-18T01:14:41.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2996989952 bytes 2024-09-18T01:14:41.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3047284736 bytes 2024-09-18T01:14:42.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3137715200 bytes 2024-09-18T01:14:42.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3217375232 bytes 2024-09-18T01:14:42.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3252488192 bytes 2024-09-18T01:14:42.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3298983936 bytes 2024-09-18T01:14:42.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3411515392 bytes 2024-09-18T01:14:42.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3437888512 bytes 2024-09-18T01:14:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3534057472 bytes 2024-09-18T01:14:43.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3555909632 bytes 2024-09-18T01:14:43.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3656280064 bytes 2024-09-18T01:14:43.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3720074240 bytes 2024-09-18T01:14:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3753392128 bytes 2024-09-18T01:14:43.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3776410624 bytes 2024-09-18T01:14:44.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3786485760 bytes 2024-09-18T01:14:44.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3790407680 bytes 2024-09-18T01:14:44.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3830153216 bytes 2024-09-18T01:14:44.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3837381632 bytes 2024-09-18T01:14:44.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3873996800 bytes 2024-09-18T01:14:44.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3946418176 bytes 2024-09-18T01:14:44.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3973450752 bytes 2024-09-18T01:14:44.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4010512384 bytes 2024-09-18T01:14:44.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4040410112 bytes 2024-09-18T01:14:45.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4109544448 bytes 2024-09-18T01:14:45.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4148704256 bytes 2024-09-18T01:14:45.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4190528512 bytes 2024-09-18T01:14:45.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_29 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 snapshot 128M 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:14:45.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_29 128M 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:14:45.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:14:45.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:45.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_29 2024-09-18T01:14:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:14:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:14:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_29 snapshot 2024-09-18T01:14:45.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:45.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:45.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:14:45.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:45.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_29 snapshot 2024-09-18T01:14:45.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:14:45.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_29 2024-09-18T01:14:45.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_29': 2024-09-18T01:14:45.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:14:45.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:14:45.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:14:45.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1363f9473b6e 2024-09-18T01:14:45.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1363f9473b6e 2024-09-18T01:14:45.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:14:45.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:14:45.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:14:45.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:14:45.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:45 2024 2024-09-18T01:14:45.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:45 2024 2024-09-18T01:14:45.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:45 2024 2024-09-18T01:14:45.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:14:45.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:14:45.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: f1b25004-306f-4390-bf36-878a869a0106 2024-09-18T01:14:45.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:14:45.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:14:45.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_29 2024-09-18T01:14:45.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:45.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:45.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:14:45.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:14:45.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=10 2024-09-18T01:14:45.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:14:45.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:14:46.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 10544128 bytes 2024-09-18T01:14:46.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 12839936 bytes 2024-09-18T01:14:46.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 72982528 bytes 2024-09-18T01:14:46.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 139787264 bytes 2024-09-18T01:14:46.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 148824064 bytes 2024-09-18T01:14:46.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 181098496 bytes 2024-09-18T01:14:46.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 254925824 bytes 2024-09-18T01:14:46.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 283216896 bytes 2024-09-18T01:14:47.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 369969152 bytes 2024-09-18T01:14:47.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 421885952 bytes 2024-09-18T01:14:47.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 443013120 bytes 2024-09-18T01:14:47.528 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 481771520 bytes 2024-09-18T01:14:47.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 499236864 bytes 2024-09-18T01:14:47.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 542493696 bytes 2024-09-18T01:14:47.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 592044032 bytes 2024-09-18T01:14:48.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 640331776 bytes 2024-09-18T01:14:48.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 659387392 bytes 2024-09-18T01:14:48.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 730782720 bytes 2024-09-18T01:14:48.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 731431936 bytes 2024-09-18T01:14:48.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 762529792 bytes 2024-09-18T01:14:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 814119936 bytes 2024-09-18T01:14:48.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 855706624 bytes 2024-09-18T01:14:48.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 899951616 bytes 2024-09-18T01:14:48.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 903603200 bytes 2024-09-18T01:14:49.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 946582528 bytes 2024-09-18T01:14:49.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1036636160 bytes 2024-09-18T01:14:49.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1064178688 bytes 2024-09-18T01:14:49.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1064777728 bytes 2024-09-18T01:14:49.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1069783040 bytes 2024-09-18T01:14:49.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1086554112 bytes 2024-09-18T01:14:49.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1117402112 bytes 2024-09-18T01:14:49.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1134747648 bytes 2024-09-18T01:14:50.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1155955712 bytes 2024-09-18T01:14:50.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1186944000 bytes 2024-09-18T01:14:50.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1251864576 bytes 2024-09-18T01:14:50.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1302157312 bytes 2024-09-18T01:14:50.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1352717312 bytes 2024-09-18T01:14:50.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1371722752 bytes 2024-09-18T01:14:50.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1432648704 bytes 2024-09-18T01:14:50.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1450032128 bytes 2024-09-18T01:14:51.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1481111552 bytes 2024-09-18T01:14:51.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1549592576 bytes 2024-09-18T01:14:51.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1604973568 bytes 2024-09-18T01:14:51.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1644044288 bytes 2024-09-18T01:14:51.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1732619264 bytes 2024-09-18T01:14:51.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1798360064 bytes 2024-09-18T01:14:51.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1840068608 bytes 2024-09-18T01:14:52.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1867917312 bytes 2024-09-18T01:14:52.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1922721792 bytes 2024-09-18T01:14:52.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1967675392 bytes 2024-09-18T01:14:52.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 1972759552 bytes 2024-09-18T01:14:52.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2018723840 bytes 2024-09-18T01:14:52.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2070764544 bytes 2024-09-18T01:14:52.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2137543680 bytes 2024-09-18T01:14:52.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2169298944 bytes 2024-09-18T01:14:53.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2232549376 bytes 2024-09-18T01:14:53.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2259714048 bytes 2024-09-18T01:14:53.338 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2281080832 bytes 2024-09-18T01:14:53.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2331233280 bytes 2024-09-18T01:14:53.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2369228800 bytes 2024-09-18T01:14:53.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2446085120 bytes 2024-09-18T01:14:53.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2501251072 bytes 2024-09-18T01:14:54.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2565525504 bytes 2024-09-18T01:14:54.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2597050368 bytes 2024-09-18T01:14:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2616885248 bytes 2024-09-18T01:14:54.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2643683328 bytes 2024-09-18T01:14:54.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2661465088 bytes 2024-09-18T01:14:54.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2698258432 bytes 2024-09-18T01:14:54.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2698876928 bytes 2024-09-18T01:14:54.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2715508736 bytes 2024-09-18T01:14:54.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2761615360 bytes 2024-09-18T01:14:55.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2834164736 bytes 2024-09-18T01:14:55.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 2851351552 bytes 2024-09-18T01:14:55.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 2893324288 bytes 2024-09-18T01:14:55.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 2936125440 bytes 2024-09-18T01:14:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3000045568 bytes 2024-09-18T01:14:55.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3048519680 bytes 2024-09-18T01:14:55.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3056129024 bytes 2024-09-18T01:14:55.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:14:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:14:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:14:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:14:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_30 2024-09-18T01:14:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 snapshot 128M 2024-09-18T01:14:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:14:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:14:55.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_30 128M 2024-09-18T01:14:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:55.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:14:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:14:55.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:14:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:14:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:14:55.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_30 2024-09-18T01:14:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:14:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:14:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:14:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_30 snapshot 2024-09-18T01:14:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:14:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:14:56.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_30 snapshot 2024-09-18T01:14:56.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:14:56.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_30 2024-09-18T01:14:56.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_30': 2024-09-18T01:14:56.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:14:56.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:14:56.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:14:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 136fc4ad236e 2024-09-18T01:14:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.136fc4ad236e 2024-09-18T01:14:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:14:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:14:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:14:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:14:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:56 2024 2024-09-18T01:14:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:56 2024 2024-09-18T01:14:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:56 2024 2024-09-18T01:14:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:14:56.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:14:56.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: d6c020f1-f8ac-4074-858c-a7dfb0ecf52e 2024-09-18T01:14:56.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:14:57.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:14:57.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_30 2024-09-18T01:14:57.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:14:57.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:14:57.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:14:57.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:14:57.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:14:57.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:14:57.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:14:57.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 10358784 bytes 2024-09-18T01:14:57.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 89733120 bytes 2024-09-18T01:14:57.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 90346496 bytes 2024-09-18T01:14:57.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 106981376 bytes 2024-09-18T01:14:57.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 218731520 bytes 2024-09-18T01:14:57.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 296680448 bytes 2024-09-18T01:14:57.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 311260160 bytes 2024-09-18T01:14:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 320959488 bytes 2024-09-18T01:14:58.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 345785344 bytes 2024-09-18T01:14:58.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 365235200 bytes 2024-09-18T01:14:58.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 427332608 bytes 2024-09-18T01:14:58.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 460619776 bytes 2024-09-18T01:14:58.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 468743168 bytes 2024-09-18T01:14:58.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 585064448 bytes 2024-09-18T01:14:58.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 635756544 bytes 2024-09-18T01:14:59.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 688013312 bytes 2024-09-18T01:14:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 716036096 bytes 2024-09-18T01:14:59.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 785630208 bytes 2024-09-18T01:14:59.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 790202368 bytes 2024-09-18T01:14:59.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 819112960 bytes 2024-09-18T01:14:59.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 841345024 bytes 2024-09-18T01:14:59.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 930455552 bytes 2024-09-18T01:15:00.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 969003008 bytes 2024-09-18T01:15:00.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1039680512 bytes 2024-09-18T01:15:00.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1042795520 bytes 2024-09-18T01:15:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1076638720 bytes 2024-09-18T01:15:00.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1100381184 bytes 2024-09-18T01:15:00.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1128418304 bytes 2024-09-18T01:15:00.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1154002944 bytes 2024-09-18T01:15:00.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1226099712 bytes 2024-09-18T01:15:01.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1251808256 bytes 2024-09-18T01:15:01.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1261961216 bytes 2024-09-18T01:15:01.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1346929664 bytes 2024-09-18T01:15:01.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1405412352 bytes 2024-09-18T01:15:01.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1508952064 bytes 2024-09-18T01:15:01.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1551520768 bytes 2024-09-18T01:15:01.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1584395264 bytes 2024-09-18T01:15:02.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1684477952 bytes 2024-09-18T01:15:02.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1721992192 bytes 2024-09-18T01:15:02.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1752748032 bytes 2024-09-18T01:15:02.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1825126400 bytes 2024-09-18T01:15:02.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1875604480 bytes 2024-09-18T01:15:02.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1915700224 bytes 2024-09-18T01:15:02.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1966322688 bytes 2024-09-18T01:15:02.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2030954496 bytes 2024-09-18T01:15:03.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2061131776 bytes 2024-09-18T01:15:03.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2072435712 bytes 2024-09-18T01:15:03.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2145280000 bytes 2024-09-18T01:15:03.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2155314176 bytes 2024-09-18T01:15:03.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2188898304 bytes 2024-09-18T01:15:03.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2235847680 bytes 2024-09-18T01:15:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2279830528 bytes 2024-09-18T01:15:04.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2344742912 bytes 2024-09-18T01:15:04.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2371575808 bytes 2024-09-18T01:15:04.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2393047040 bytes 2024-09-18T01:15:04.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2447756288 bytes 2024-09-18T01:15:04.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2474625024 bytes 2024-09-18T01:15:04.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2497324032 bytes 2024-09-18T01:15:04.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2532521984 bytes 2024-09-18T01:15:04.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2595268608 bytes 2024-09-18T01:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2612856832 bytes 2024-09-18T01:15:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2626031616 bytes 2024-09-18T01:15:05.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2667316224 bytes 2024-09-18T01:15:05.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2708203520 bytes 2024-09-18T01:15:05.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2771478528 bytes 2024-09-18T01:15:05.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2772092928 bytes 2024-09-18T01:15:05.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2798017536 bytes 2024-09-18T01:15:05.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2828256256 bytes 2024-09-18T01:15:06.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2881787904 bytes 2024-09-18T01:15:06.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2882409472 bytes 2024-09-18T01:15:06.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2940338176 bytes 2024-09-18T01:15:06.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2960657408 bytes 2024-09-18T01:15:06.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3034858496 bytes 2024-09-18T01:15:06.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3087251456 bytes 2024-09-18T01:15:06.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3128579072 bytes 2024-09-18T01:15:06.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3154655232 bytes 2024-09-18T01:15:07.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3184754688 bytes 2024-09-18T01:15:07.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3223470080 bytes 2024-09-18T01:15:07.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3257853952 bytes 2024-09-18T01:15:07.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3258456064 bytes 2024-09-18T01:15:07.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3328844800 bytes 2024-09-18T01:15:07.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3358402560 bytes 2024-09-18T01:15:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3405386752 bytes 2024-09-18T01:15:08.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3490408448 bytes 2024-09-18T01:15:08.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3539773440 bytes 2024-09-18T01:15:08.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3564748800 bytes 2024-09-18T01:15:08.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3612489728 bytes 2024-09-18T01:15:08.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3635102720 bytes 2024-09-18T01:15:08.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3664946176 bytes 2024-09-18T01:15:08.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3751451648 bytes 2024-09-18T01:15:08.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3781923840 bytes 2024-09-18T01:15:09.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3790187520 bytes 2024-09-18T01:15:09.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3845875712 bytes 2024-09-18T01:15:09.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3890698240 bytes 2024-09-18T01:15:09.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 3955579904 bytes 2024-09-18T01:15:09.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 3956198400 bytes 2024-09-18T01:15:09.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 3956816896 bytes 2024-09-18T01:15:09.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4043305984 bytes 2024-09-18T01:15:09.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4097345536 bytes 2024-09-18T01:15:10.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4113262592 bytes 2024-09-18T01:15:10.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4176407552 bytes 2024-09-18T01:15:10.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4211017728 bytes 2024-09-18T01:15:10.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4237313024 bytes 2024-09-18T01:15:10.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4270512128 bytes 2024-09-18T01:15:10.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4290183168 bytes 2024-09-18T01:15:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4367532032 bytes 2024-09-18T01:15:10.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4421905408 bytes 2024-09-18T01:15:11.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:15:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:15:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:15:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:15:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_31 2024-09-18T01:15:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 snapshot 128M 2024-09-18T01:15:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:15:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:11.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_31 128M 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:11.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_31 2024-09-18T01:15:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:15:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:15:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_31 snapshot 2024-09-18T01:15:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:11.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:15:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_31 snapshot 2024-09-18T01:15:11.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:15:11.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_31 2024-09-18T01:15:11.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_31': 2024-09-18T01:15:11.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:15:11.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:15:11.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 137b1ed2762a 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.137b1ed2762a 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:15:11 2024 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:15:11 2024 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:15:11 2024 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:15:11.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 753ffe95-b234-4f46-a463-e642f6aa8751 2024-09-18T01:15:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:15:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:15:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_31 2024-09-18T01:15:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:15:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:15:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=13 2024-09-18T01:15:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:15:11.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:15:11.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 65668096 bytes 2024-09-18T01:15:11.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 92510208 bytes 2024-09-18T01:15:11.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 142972928 bytes 2024-09-18T01:15:11.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 145829888 bytes 2024-09-18T01:15:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 151954432 bytes 2024-09-18T01:15:12.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 167121920 bytes 2024-09-18T01:15:12.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 232754176 bytes 2024-09-18T01:15:12.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 242959360 bytes 2024-09-18T01:15:12.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 304125952 bytes 2024-09-18T01:15:12.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 305176576 bytes 2024-09-18T01:15:12.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 342218752 bytes 2024-09-18T01:15:12.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 380478464 bytes 2024-09-18T01:15:12.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 404861952 bytes 2024-09-18T01:15:13.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 463998976 bytes 2024-09-18T01:15:13.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 534535168 bytes 2024-09-18T01:15:13.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 553895936 bytes 2024-09-18T01:15:13.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 554518528 bytes 2024-09-18T01:15:13.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 606321664 bytes 2024-09-18T01:15:13.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 608717824 bytes 2024-09-18T01:15:13.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 639121408 bytes 2024-09-18T01:15:13.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 645080064 bytes 2024-09-18T01:15:14.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 683779072 bytes 2024-09-18T01:15:14.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 721921024 bytes 2024-09-18T01:15:14.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 734418944 bytes 2024-09-18T01:15:14.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 788069376 bytes 2024-09-18T01:15:14.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 896872448 bytes 2024-09-18T01:15:14.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 935689216 bytes 2024-09-18T01:15:14.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 965730304 bytes 2024-09-18T01:15:14.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1010928640 bytes 2024-09-18T01:15:15.087 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1039900672 bytes 2024-09-18T01:15:15.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1118267392 bytes 2024-09-18T01:15:15.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1221690368 bytes 2024-09-18T01:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1273488384 bytes 2024-09-18T01:15:15.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1324939264 bytes 2024-09-18T01:15:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1374575616 bytes 2024-09-18T01:15:15.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1427547136 bytes 2024-09-18T01:15:16.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1518646272 bytes 2024-09-18T01:15:16.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1542330368 bytes 2024-09-18T01:15:16.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1607397376 bytes 2024-09-18T01:15:16.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1624547328 bytes 2024-09-18T01:15:16.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1692268544 bytes 2024-09-18T01:15:16.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1751736320 bytes 2024-09-18T01:15:16.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1771749376 bytes 2024-09-18T01:15:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1784323072 bytes 2024-09-18T01:15:17.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1884742656 bytes 2024-09-18T01:15:17.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1918904320 bytes 2024-09-18T01:15:17.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1957763072 bytes 2024-09-18T01:15:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1963452416 bytes 2024-09-18T01:15:17.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2039200768 bytes 2024-09-18T01:15:17.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2058236928 bytes 2024-09-18T01:15:17.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2177755136 bytes 2024-09-18T01:15:18.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2239857664 bytes 2024-09-18T01:15:18.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2283889664 bytes 2024-09-18T01:15:18.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2367497216 bytes 2024-09-18T01:15:18.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2485569536 bytes 2024-09-18T01:15:18.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2568876032 bytes 2024-09-18T01:15:18.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2578695168 bytes 2024-09-18T01:15:19.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2631778304 bytes 2024-09-18T01:15:19.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2688020480 bytes 2024-09-18T01:15:19.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2729191424 bytes 2024-09-18T01:15:19.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2770723840 bytes 2024-09-18T01:15:19.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2799720448 bytes 2024-09-18T01:15:19.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2833339392 bytes 2024-09-18T01:15:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2859172864 bytes 2024-09-18T01:15:20.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2961862656 bytes 2024-09-18T01:15:20.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 3012285440 bytes 2024-09-18T01:15:20.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3072031744 bytes 2024-09-18T01:15:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3085453312 bytes 2024-09-18T01:15:20.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3131750400 bytes 2024-09-18T01:15:20.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3192176640 bytes 2024-09-18T01:15:20.783 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3216485376 bytes 2024-09-18T01:15:20.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3255075840 bytes 2024-09-18T01:15:21.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3326116864 bytes 2024-09-18T01:15:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3340052480 bytes 2024-09-18T01:15:21.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3369083904 bytes 2024-09-18T01:15:21.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3369705472 bytes 2024-09-18T01:15:21.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3433735168 bytes 2024-09-18T01:15:21.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3438105600 bytes 2024-09-18T01:15:21.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3476226048 bytes 2024-09-18T01:15:21.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3516880896 bytes 2024-09-18T01:15:21.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3547517952 bytes 2024-09-18T01:15:22.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3631915008 bytes 2024-09-18T01:15:22.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3656047616 bytes 2024-09-18T01:15:22.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3726673920 bytes 2024-09-18T01:15:22.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3728524288 bytes 2024-09-18T01:15:22.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3841459200 bytes 2024-09-18T01:15:22.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3854480384 bytes 2024-09-18T01:15:22.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3862529024 bytes 2024-09-18T01:15:23.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3909982208 bytes 2024-09-18T01:15:23.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3942020096 bytes 2024-09-18T01:15:23.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3981923328 bytes 2024-09-18T01:15:23.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4039939072 bytes 2024-09-18T01:15:23.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4072128512 bytes 2024-09-18T01:15:23.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4156718080 bytes 2024-09-18T01:15:23.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4212990976 bytes 2024-09-18T01:15:24.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4266613760 bytes 2024-09-18T01:15:24.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4309089280 bytes 2024-09-18T01:15:24.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4363834368 bytes 2024-09-18T01:15:24.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4439386112 bytes 2024-09-18T01:15:24.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:15:24.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:15:24.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:15:24.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:24.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:15:24.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_32 2024-09-18T01:15:24.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 snapshot 128M 2024-09-18T01:15:24.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_32 128M 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:24.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:24.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:15:24.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:24.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:15:24.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:15:24.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:15:24.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:24.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_32 2024-09-18T01:15:24.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:15:24.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:24.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:15:24.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:24.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_32 snapshot 2024-09-18T01:15:24.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:24.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:24.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:15:24.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:24.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_32 snapshot 2024-09-18T01:15:25.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:15:25.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_32 2024-09-18T01:15:25.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_32': 2024-09-18T01:15:25.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:15:25.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:15:25.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13879770b099 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13879770b099 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:15:24 2024 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:15:24 2024 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:15:24 2024 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:15:25.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 884d933d-d88c-4657-97ca-5aa2e3991921 2024-09-18T01:15:25.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:15:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:15:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_32 2024-09-18T01:15:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:15:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:15:25.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=8 2024-09-18T01:15:25.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:15:25.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:15:25.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 35377152 bytes 2024-09-18T01:15:25.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 75246592 bytes 2024-09-18T01:15:25.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 75864064 bytes 2024-09-18T01:15:25.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 129268736 bytes 2024-09-18T01:15:26.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 162424832 bytes 2024-09-18T01:15:26.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 189594624 bytes 2024-09-18T01:15:26.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 212304896 bytes 2024-09-18T01:15:26.471 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 259075072 bytes 2024-09-18T01:15:26.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 330227712 bytes 2024-09-18T01:15:26.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 369061888 bytes 2024-09-18T01:15:26.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 439541760 bytes 2024-09-18T01:15:27.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 487198720 bytes 2024-09-18T01:15:27.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 535398400 bytes 2024-09-18T01:15:27.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 556034048 bytes 2024-09-18T01:15:27.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 560647168 bytes 2024-09-18T01:15:27.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 659508224 bytes 2024-09-18T01:15:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 704826368 bytes 2024-09-18T01:15:27.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 733838336 bytes 2024-09-18T01:15:27.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 806176768 bytes 2024-09-18T01:15:28.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 856472576 bytes 2024-09-18T01:15:28.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 864019456 bytes 2024-09-18T01:15:28.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 913928192 bytes 2024-09-18T01:15:28.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 983537664 bytes 2024-09-18T01:15:28.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1000721408 bytes 2024-09-18T01:15:28.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1063429120 bytes 2024-09-18T01:15:28.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1119787008 bytes 2024-09-18T01:15:29.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1137494016 bytes 2024-09-18T01:15:29.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1172466688 bytes 2024-09-18T01:15:29.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1210218496 bytes 2024-09-18T01:15:29.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1273323520 bytes 2024-09-18T01:15:29.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1286687744 bytes 2024-09-18T01:15:29.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1313744896 bytes 2024-09-18T01:15:29.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1370784768 bytes 2024-09-18T01:15:29.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1424415744 bytes 2024-09-18T01:15:30.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1435228160 bytes 2024-09-18T01:15:30.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1472727040 bytes 2024-09-18T01:15:30.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1495376896 bytes 2024-09-18T01:15:30.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1543433216 bytes 2024-09-18T01:15:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1578798080 bytes 2024-09-18T01:15:30.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1613426688 bytes 2024-09-18T01:15:30.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1673143296 bytes 2024-09-18T01:15:30.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1718679552 bytes 2024-09-18T01:15:30.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1719290880 bytes 2024-09-18T01:15:31.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1775075328 bytes 2024-09-18T01:15:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1866474496 bytes 2024-09-18T01:15:31.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1895172096 bytes 2024-09-18T01:15:31.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1918631936 bytes 2024-09-18T01:15:31.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2020637696 bytes 2024-09-18T01:15:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2050958336 bytes 2024-09-18T01:15:31.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2080829440 bytes 2024-09-18T01:15:32.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2093796352 bytes 2024-09-18T01:15:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2100374528 bytes 2024-09-18T01:15:32.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2177702912 bytes 2024-09-18T01:15:32.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2209132544 bytes 2024-09-18T01:15:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2292445184 bytes 2024-09-18T01:15:32.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2322885632 bytes 2024-09-18T01:15:32.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2340391936 bytes 2024-09-18T01:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2384428032 bytes 2024-09-18T01:15:33.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2390250496 bytes 2024-09-18T01:15:33.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2403390464 bytes 2024-09-18T01:15:33.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2413149184 bytes 2024-09-18T01:15:33.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2422448128 bytes 2024-09-18T01:15:33.436 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:15:33.432+0000 7f80d5d16640 -1 reset not still connected to 0x564178a50c30 2024-09-18T01:15:33.436 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:15:33.432+0000 7f80d5d16640 -1 reset not still connected to 0x564178a51040 2024-09-18T01:15:33.436 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:15:33.432+0000 7f80d5d16640 -1 reset not still connected to 0x564194f996c0 2024-09-18T01:15:33.436 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:15:33.432+0000 7f80d5d16640 -1 reset not still connected to 0x5641a10cf2b0 2024-09-18T01:15:33.490 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:15:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:15:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:15:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:15:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_33 2024-09-18T01:15:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 snapshot 128M 2024-09-18T01:15:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:33.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:15:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:15:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_33 128M 2024-09-18T01:15:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:33.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:33.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:15:33.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:33.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:15:33.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:15:33.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:15:33.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:33.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_33 2024-09-18T01:15:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:15:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:15:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_33 snapshot 2024-09-18T01:15:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:15:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_33 snapshot 2024-09-18T01:15:34.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:15:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_33 2024-09-18T01:15:34.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_33': 2024-09-18T01:15:34.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:15:34.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:15:34.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:15:34.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13938be3f846 2024-09-18T01:15:34.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13938be3f846 2024-09-18T01:15:34.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:15:34.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:15:34.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:15:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:15:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:15:33 2024 2024-09-18T01:15:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:15:33 2024 2024-09-18T01:15:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:15:33 2024 2024-09-18T01:15:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:15:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:15:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 4d961941-4020-4df3-821f-ef25e429389f 2024-09-18T01:15:34.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:15:34.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:15:34.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_33 2024-09-18T01:15:34.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:34.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:34.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:15:34.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:15:34.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=5 2024-09-18T01:15:34.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:15:34.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:15:34.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 54532096 bytes 2024-09-18T01:15:34.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 79250432 bytes 2024-09-18T01:15:34.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 100684800 bytes 2024-09-18T01:15:35.021 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 132680704 bytes 2024-09-18T01:15:35.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 185015296 bytes 2024-09-18T01:15:35.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 225106944 bytes 2024-09-18T01:15:35.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 252049408 bytes 2024-09-18T01:15:35.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 265325568 bytes 2024-09-18T01:15:35.503 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-18T01:15:35.503 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-18T01:15:35.504 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-18T01:15:35.505 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:15:35.503+0000 7fdd1d1c8640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 45648) UID: 1000 2024-09-18T01:15:35.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 312432640 bytes 2024-09-18T01:15:35.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 345249792 bytes 2024-09-18T01:15:35.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 359091200 bytes 2024-09-18T01:15:36.087 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 432090112 bytes 2024-09-18T01:15:36.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 471731200 bytes 2024-09-18T01:15:36.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 483136512 bytes 2024-09-18T01:15:36.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 546159616 bytes 2024-09-18T01:15:36.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 592379904 bytes 2024-09-18T01:15:36.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 695873536 bytes 2024-09-18T01:15:36.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 705004544 bytes 2024-09-18T01:15:36.967 INFO:tasks.ceph.cluster2.osd.1.smithi150.stderr:2024-09-18T01:15:36.965+0000 7f3bf08d2640 -1 reset not still connected to 0x55b70a01e9c0 2024-09-18T01:15:37.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 737924096 bytes 2024-09-18T01:15:37.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 793199616 bytes 2024-09-18T01:15:37.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 835984384 bytes 2024-09-18T01:15:37.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 888852480 bytes 2024-09-18T01:15:37.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 909441024 bytes 2024-09-18T01:15:37.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 929099776 bytes 2024-09-18T01:15:37.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 937655296 bytes 2024-09-18T01:15:37.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 943509504 bytes 2024-09-18T01:15:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 958942208 bytes 2024-09-18T01:15:38.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1017028608 bytes 2024-09-18T01:15:38.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1069329408 bytes 2024-09-18T01:15:38.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1107383296 bytes 2024-09-18T01:15:38.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1182552064 bytes 2024-09-18T01:15:38.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1231081472 bytes 2024-09-18T01:15:38.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1279455232 bytes 2024-09-18T01:15:38.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1280066560 bytes 2024-09-18T01:15:39.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1315410944 bytes 2024-09-18T01:15:39.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1340232704 bytes 2024-09-18T01:15:39.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1382326272 bytes 2024-09-18T01:15:39.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1390980096 bytes 2024-09-18T01:15:39.504 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-18T01:15:39.504 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:15:39.505 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-18T01:15:39.505 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-18T01:15:39.505 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-18T01:15:39.508 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-18T01:15:39.508 INFO:tasks.rbd_mirror_thrash:waiting for 108 secs before thrashing 2024-09-18T01:15:39.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:15:39.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:15:39.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:15:39.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:39.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_34 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 snapshot 128M 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_34 128M 2024-09-18T01:15:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:39.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:39.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:39.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:39.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:15:39.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:39.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:15:39.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:15:39.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:15:39.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:39.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_34 2024-09-18T01:15:39.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:15:39.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:15:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_34 snapshot 2024-09-18T01:15:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:15:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:39.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_34 snapshot 2024-09-18T01:15:40.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:15:40.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_34 2024-09-18T01:15:40.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_34': 2024-09-18T01:15:40.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:15:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:15:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:15:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 139ff16c4ac8 2024-09-18T01:15:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.139ff16c4ac8 2024-09-18T01:15:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:15:40.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:15:40.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:15:40.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:15:40.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:15:39 2024 2024-09-18T01:15:40.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:15:39 2024 2024-09-18T01:15:40.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:15:39 2024 2024-09-18T01:15:40.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:15:40.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:15:40.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 4770a819-7931-426e-a8e9-0d6c6ec6d2de 2024-09-18T01:15:40.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:15:40.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:15:40.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_34 2024-09-18T01:15:40.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:40.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:40.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:15:40.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:15:40.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=5 2024-09-18T01:15:40.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:15:40.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:15:40.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 27561984 bytes 2024-09-18T01:15:40.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 75557888 bytes 2024-09-18T01:15:40.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 81166336 bytes 2024-09-18T01:15:40.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 100765696 bytes 2024-09-18T01:15:41.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 246354944 bytes 2024-09-18T01:15:41.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 270067712 bytes 2024-09-18T01:15:41.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 369645568 bytes 2024-09-18T01:15:41.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 430287872 bytes 2024-09-18T01:15:41.815 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 503228416 bytes 2024-09-18T01:15:41.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 537947136 bytes 2024-09-18T01:15:42.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 641225728 bytes 2024-09-18T01:15:42.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 703696896 bytes 2024-09-18T01:15:42.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 720188416 bytes 2024-09-18T01:15:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 807307264 bytes 2024-09-18T01:15:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 833661952 bytes 2024-09-18T01:15:42.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 862889984 bytes 2024-09-18T01:15:42.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 870067200 bytes 2024-09-18T01:15:42.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 878026752 bytes 2024-09-18T01:15:43.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 916921344 bytes 2024-09-18T01:15:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 952594432 bytes 2024-09-18T01:15:43.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1004119040 bytes 2024-09-18T01:15:43.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1093755904 bytes 2024-09-18T01:15:43.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1158439936 bytes 2024-09-18T01:15:43.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1168332800 bytes 2024-09-18T01:15:43.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1183395840 bytes 2024-09-18T01:15:43.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1184013312 bytes 2024-09-18T01:15:44.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1276338176 bytes 2024-09-18T01:15:44.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1315948544 bytes 2024-09-18T01:15:44.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1385422848 bytes 2024-09-18T01:15:44.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1410166784 bytes 2024-09-18T01:15:44.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1422564352 bytes 2024-09-18T01:15:44.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1460676608 bytes 2024-09-18T01:15:44.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1477089280 bytes 2024-09-18T01:15:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1586117632 bytes 2024-09-18T01:15:45.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1601932288 bytes 2024-09-18T01:15:45.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1619385344 bytes 2024-09-18T01:15:45.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1669582848 bytes 2024-09-18T01:15:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:15:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:15:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:15:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:15:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_35 2024-09-18T01:15:45.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 snapshot 128M 2024-09-18T01:15:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:15:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:15:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:45.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_35 128M 2024-09-18T01:15:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:15:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:45.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:15:45.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:15:45.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:15:45.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:45.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_35 2024-09-18T01:15:45.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:15:45.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:45.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:15:45.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:45.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_35 snapshot 2024-09-18T01:15:45.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:15:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:45.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_35 snapshot 2024-09-18T01:15:46.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:15:46.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_35 2024-09-18T01:15:46.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_35': 2024-09-18T01:15:46.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:15:46.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:15:46.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:15:46.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13b3a89f56a5 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13b3a89f56a5 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:15:45 2024 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:15:45 2024 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:15:45 2024 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 855097c3-5440-41cd-a767-29e218b7cf40 2024-09-18T01:15:46.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:15:46.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:15:46.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_35 2024-09-18T01:15:46.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:46.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:46.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:15:46.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:15:46.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=7 2024-09-18T01:15:46.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:15:46.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:15:46.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 42893312 bytes 2024-09-18T01:15:46.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 85429248 bytes 2024-09-18T01:15:46.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 121700352 bytes 2024-09-18T01:15:47.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 129543168 bytes 2024-09-18T01:15:47.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 173200384 bytes 2024-09-18T01:15:47.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 200864768 bytes 2024-09-18T01:15:47.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 234923008 bytes 2024-09-18T01:15:47.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 255015936 bytes 2024-09-18T01:15:47.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 282320896 bytes 2024-09-18T01:15:47.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 314249216 bytes 2024-09-18T01:15:47.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 365489152 bytes 2024-09-18T01:15:48.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 415660032 bytes 2024-09-18T01:15:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 459060224 bytes 2024-09-18T01:15:48.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 480339968 bytes 2024-09-18T01:15:48.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 498508800 bytes 2024-09-18T01:15:48.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 579939328 bytes 2024-09-18T01:15:48.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 615520256 bytes 2024-09-18T01:15:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 682757120 bytes 2024-09-18T01:15:48.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 692898816 bytes 2024-09-18T01:15:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 710027264 bytes 2024-09-18T01:15:49.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 766900224 bytes 2024-09-18T01:15:49.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 784678912 bytes 2024-09-18T01:15:49.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 826501120 bytes 2024-09-18T01:15:49.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 827129856 bytes 2024-09-18T01:15:49.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 845517824 bytes 2024-09-18T01:15:49.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 846119936 bytes 2024-09-18T01:15:49.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 900396032 bytes 2024-09-18T01:15:49.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 959909888 bytes 2024-09-18T01:15:50.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 996383744 bytes 2024-09-18T01:15:50.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1041748992 bytes 2024-09-18T01:15:50.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1111703552 bytes 2024-09-18T01:15:50.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1153928192 bytes 2024-09-18T01:15:50.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1180737536 bytes 2024-09-18T01:15:50.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1217506304 bytes 2024-09-18T01:15:50.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1218573312 bytes 2024-09-18T01:15:50.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1276312576 bytes 2024-09-18T01:15:51.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1301707776 bytes 2024-09-18T01:15:51.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1319333888 bytes 2024-09-18T01:15:51.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1354745856 bytes 2024-09-18T01:15:51.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1393829888 bytes 2024-09-18T01:15:51.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1444398080 bytes 2024-09-18T01:15:51.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1461225472 bytes 2024-09-18T01:15:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1512201216 bytes 2024-09-18T01:15:51.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1531441152 bytes 2024-09-18T01:15:52.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1608995840 bytes 2024-09-18T01:15:52.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1652013056 bytes 2024-09-18T01:15:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1714027520 bytes 2024-09-18T01:15:52.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1799604224 bytes 2024-09-18T01:15:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1855521792 bytes 2024-09-18T01:15:52.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1893757952 bytes 2024-09-18T01:15:52.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 1911913472 bytes 2024-09-18T01:15:52.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 1942553600 bytes 2024-09-18T01:15:53.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 1958786048 bytes 2024-09-18T01:15:53.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2020771840 bytes 2024-09-18T01:15:53.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2078256128 bytes 2024-09-18T01:15:53.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2091963392 bytes 2024-09-18T01:15:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:15:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:15:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:15:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_36 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 snapshot 128M 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:15:53.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_36 128M 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:15:53.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_36 2024-09-18T01:15:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:15:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:53.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:15:53.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:15:53.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_36 snapshot 2024-09-18T01:15:53.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:53.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:53.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:15:53.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:15:53.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_36 snapshot 2024-09-18T01:15:54.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:15:54.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_36 2024-09-18T01:15:54.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_36': 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13bc30ec6eca 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13bc30ec6eca 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:15:53 2024 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:15:53 2024 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:15:53 2024 2024-09-18T01:15:54.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:15:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:15:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 3ee14da1-57b2-4d60-bd27-8af0515acf51 2024-09-18T01:15:54.548 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:15:54.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:15:54.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_36 2024-09-18T01:15:54.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:15:54.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:15:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:15:54.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:15:54.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=9 2024-09-18T01:15:54.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:15:54.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:15:54.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 44132352 bytes 2024-09-18T01:15:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 82659328 bytes 2024-09-18T01:15:54.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 145716224 bytes 2024-09-18T01:15:55.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 146331648 bytes 2024-09-18T01:15:55.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 171421696 bytes 2024-09-18T01:15:55.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 247062528 bytes 2024-09-18T01:15:55.434 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-18T01:15:55.434 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-18T01:15:55.435 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-09-18T01:15:55.436 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:15:55.435+0000 7f816647b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 47470) UID: 1000 2024-09-18T01:15:55.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 293314560 bytes 2024-09-18T01:15:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 348843008 bytes 2024-09-18T01:15:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 349464576 bytes 2024-09-18T01:15:55.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 350096384 bytes 2024-09-18T01:15:55.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 398255104 bytes 2024-09-18T01:15:56.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 454440960 bytes 2024-09-18T01:15:56.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 547255296 bytes 2024-09-18T01:15:56.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 592293888 bytes 2024-09-18T01:15:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 657378304 bytes 2024-09-18T01:15:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 707649536 bytes 2024-09-18T01:15:56.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 738957312 bytes 2024-09-18T01:15:56.919 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 792315904 bytes 2024-09-18T01:15:57.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 828903424 bytes 2024-09-18T01:15:57.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 897921024 bytes 2024-09-18T01:15:57.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 925383680 bytes 2024-09-18T01:15:57.435 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-18T01:15:57.435 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:15:57.435 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-18T01:15:57.436 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-18T01:15:57.436 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-18T01:15:57.439 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-18T01:15:57.439 INFO:tasks.rbd_mirror_thrash:waiting for 77 secs before thrashing 2024-09-18T01:15:57.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1008881664 bytes 2024-09-18T01:15:57.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1025169408 bytes 2024-09-18T01:15:57.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1042361344 bytes 2024-09-18T01:15:57.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1093106688 bytes 2024-09-18T01:15:57.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1139591168 bytes 2024-09-18T01:15:58.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1197580288 bytes 2024-09-18T01:15:58.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1241333760 bytes 2024-09-18T01:15:58.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1280393216 bytes 2024-09-18T01:15:58.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1284752384 bytes 2024-09-18T01:15:58.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1310206976 bytes 2024-09-18T01:15:58.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1360633856 bytes 2024-09-18T01:15:58.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1369044992 bytes 2024-09-18T01:15:58.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1429586944 bytes 2024-09-18T01:15:59.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1468952576 bytes 2024-09-18T01:15:59.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1500670976 bytes 2024-09-18T01:15:59.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1533746176 bytes 2024-09-18T01:15:59.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1558378496 bytes 2024-09-18T01:15:59.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1623365632 bytes 2024-09-18T01:15:59.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1669127168 bytes 2024-09-18T01:15:59.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1690165248 bytes 2024-09-18T01:15:59.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1732170752 bytes 2024-09-18T01:16:00.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1796307968 bytes 2024-09-18T01:16:00.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1847251968 bytes 2024-09-18T01:16:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1863994368 bytes 2024-09-18T01:16:00.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1939454976 bytes 2024-09-18T01:16:00.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1941026816 bytes 2024-09-18T01:16:00.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1985056768 bytes 2024-09-18T01:16:00.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2014457856 bytes 2024-09-18T01:16:00.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2039975936 bytes 2024-09-18T01:16:01.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2095040512 bytes 2024-09-18T01:16:01.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2122619904 bytes 2024-09-18T01:16:01.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2142125056 bytes 2024-09-18T01:16:01.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2173877248 bytes 2024-09-18T01:16:01.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2201682944 bytes 2024-09-18T01:16:01.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2298619904 bytes 2024-09-18T01:16:01.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2348486656 bytes 2024-09-18T01:16:02.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2365939712 bytes 2024-09-18T01:16:02.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2462661632 bytes 2024-09-18T01:16:02.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2508441600 bytes 2024-09-18T01:16:02.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2551919616 bytes 2024-09-18T01:16:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2577315840 bytes 2024-09-18T01:16:02.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2614835200 bytes 2024-09-18T01:16:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2650354688 bytes 2024-09-18T01:16:03.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2680565760 bytes 2024-09-18T01:16:03.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2759252992 bytes 2024-09-18T01:16:03.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2768306176 bytes 2024-09-18T01:16:03.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2810136576 bytes 2024-09-18T01:16:03.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:16:03.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:16:03.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:16:03.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:03.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:16:03.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_37 2024-09-18T01:16:03.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 snapshot 128M 2024-09-18T01:16:03.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:03.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:03.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_37 128M 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:03.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:16:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:16:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:16:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_37 2024-09-18T01:16:03.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:16:03.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:03.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:16:03.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_37 snapshot 2024-09-18T01:16:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:16:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:03.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_37 snapshot 2024-09-18T01:16:04.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:16:04.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_37 2024-09-18T01:16:04.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_37': 2024-09-18T01:16:04.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:16:04.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:16:04.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:16:04.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13cb85b82be1 2024-09-18T01:16:04.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13cb85b82be1 2024-09-18T01:16:04.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:16:04.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:16:04.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:16:04.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:16:04.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:03 2024 2024-09-18T01:16:04.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:03 2024 2024-09-18T01:16:04.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:03 2024 2024-09-18T01:16:04.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:16:04.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:16:04.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 7af955aa-6145-46fc-be77-ed5d53ea99dd 2024-09-18T01:16:04.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:16:04.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:16:04.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_37 2024-09-18T01:16:04.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:04.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:04.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:16:04.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:16:04.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:16:04.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:16:04.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:16:04.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 54478848 bytes 2024-09-18T01:16:04.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 65654784 bytes 2024-09-18T01:16:04.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 115217408 bytes 2024-09-18T01:16:05.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 123833344 bytes 2024-09-18T01:16:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 159010816 bytes 2024-09-18T01:16:05.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 210178048 bytes 2024-09-18T01:16:05.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 241743872 bytes 2024-09-18T01:16:05.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 284090368 bytes 2024-09-18T01:16:05.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 339815424 bytes 2024-09-18T01:16:05.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 389646336 bytes 2024-09-18T01:16:05.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 445735936 bytes 2024-09-18T01:16:06.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 546643968 bytes 2024-09-18T01:16:06.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 592507904 bytes 2024-09-18T01:16:06.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 597301248 bytes 2024-09-18T01:16:06.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 637590528 bytes 2024-09-18T01:16:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 681742336 bytes 2024-09-18T01:16:06.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 715294720 bytes 2024-09-18T01:16:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 751638528 bytes 2024-09-18T01:16:07.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 773124096 bytes 2024-09-18T01:16:07.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 797968384 bytes 2024-09-18T01:16:07.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 847878144 bytes 2024-09-18T01:16:07.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 895598592 bytes 2024-09-18T01:16:07.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1002070016 bytes 2024-09-18T01:16:07.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1077062656 bytes 2024-09-18T01:16:07.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1102288896 bytes 2024-09-18T01:16:07.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1128590336 bytes 2024-09-18T01:16:08.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1189103616 bytes 2024-09-18T01:16:08.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1237602304 bytes 2024-09-18T01:16:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1258788864 bytes 2024-09-18T01:16:08.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1303168000 bytes 2024-09-18T01:16:08.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1371153408 bytes 2024-09-18T01:16:08.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1406881792 bytes 2024-09-18T01:16:08.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1407492096 bytes 2024-09-18T01:16:09.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1464031232 bytes 2024-09-18T01:16:09.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1507459072 bytes 2024-09-18T01:16:09.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1540474880 bytes 2024-09-18T01:16:09.391 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1584400384 bytes 2024-09-18T01:16:09.490 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1597618176 bytes 2024-09-18T01:16:09.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1620995072 bytes 2024-09-18T01:16:09.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1623081984 bytes 2024-09-18T01:16:09.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1701015552 bytes 2024-09-18T01:16:09.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1733059584 bytes 2024-09-18T01:16:10.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1802214400 bytes 2024-09-18T01:16:10.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1867588608 bytes 2024-09-18T01:16:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1881253888 bytes 2024-09-18T01:16:10.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1941620736 bytes 2024-09-18T01:16:10.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1963240448 bytes 2024-09-18T01:16:10.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1978258432 bytes 2024-09-18T01:16:10.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2057376768 bytes 2024-09-18T01:16:11.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2101565440 bytes 2024-09-18T01:16:11.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2116548608 bytes 2024-09-18T01:16:11.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2170551296 bytes 2024-09-18T01:16:11.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2260247552 bytes 2024-09-18T01:16:11.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2272908288 bytes 2024-09-18T01:16:11.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2312626176 bytes 2024-09-18T01:16:11.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2365586432 bytes 2024-09-18T01:16:11.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2383936512 bytes 2024-09-18T01:16:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2425391104 bytes 2024-09-18T01:16:12.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2467854336 bytes 2024-09-18T01:16:12.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2489756672 bytes 2024-09-18T01:16:12.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2530188288 bytes 2024-09-18T01:16:12.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2538856448 bytes 2024-09-18T01:16:12.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2627429376 bytes 2024-09-18T01:16:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2669145088 bytes 2024-09-18T01:16:13.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2704770048 bytes 2024-09-18T01:16:13.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2729242624 bytes 2024-09-18T01:16:13.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2744398848 bytes 2024-09-18T01:16:13.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2774804480 bytes 2024-09-18T01:16:13.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2782560256 bytes 2024-09-18T01:16:13.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2852327424 bytes 2024-09-18T01:16:13.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2915556352 bytes 2024-09-18T01:16:13.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2928987136 bytes 2024-09-18T01:16:13.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 2958862336 bytes 2024-09-18T01:16:14.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 2983645184 bytes 2024-09-18T01:16:14.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3015037952 bytes 2024-09-18T01:16:14.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3039965184 bytes 2024-09-18T01:16:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3116309504 bytes 2024-09-18T01:16:14.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3140151296 bytes 2024-09-18T01:16:14.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3160979456 bytes 2024-09-18T01:16:14.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3222837248 bytes 2024-09-18T01:16:14.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3255649280 bytes 2024-09-18T01:16:15.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3297997824 bytes 2024-09-18T01:16:15.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3304154112 bytes 2024-09-18T01:16:15.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3376218112 bytes 2024-09-18T01:16:15.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3435852800 bytes 2024-09-18T01:16:15.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3482500096 bytes 2024-09-18T01:16:15.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3552896000 bytes 2024-09-18T01:16:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3582825472 bytes 2024-09-18T01:16:15.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3607387136 bytes 2024-09-18T01:16:16.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3649873920 bytes 2024-09-18T01:16:16.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3699682304 bytes 2024-09-18T01:16:16.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3717835776 bytes 2024-09-18T01:16:16.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3776796672 bytes 2024-09-18T01:16:16.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3807607808 bytes 2024-09-18T01:16:16.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 3808188416 bytes 2024-09-18T01:16:16.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 3840217088 bytes 2024-09-18T01:16:16.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 3868559360 bytes 2024-09-18T01:16:17.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 3883533312 bytes 2024-09-18T01:16:17.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 3913380864 bytes 2024-09-18T01:16:17.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 3945520128 bytes 2024-09-18T01:16:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 3947268096 bytes 2024-09-18T01:16:17.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 3990426624 bytes 2024-09-18T01:16:17.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4040535040 bytes 2024-09-18T01:16:17.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4088889344 bytes 2024-09-18T01:16:17.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4124693504 bytes 2024-09-18T01:16:17.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4165982208 bytes 2024-09-18T01:16:18.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4218646528 bytes 2024-09-18T01:16:18.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4259883008 bytes 2024-09-18T01:16:18.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4308037632 bytes 2024-09-18T01:16:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 4361857024 bytes 2024-09-18T01:16:18.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:16:18.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:16:18.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:16:18.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:18.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:16:18.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_38 2024-09-18T01:16:18.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 snapshot 128M 2024-09-18T01:16:18.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:18.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:18.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:18.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:18.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:16:18.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:18.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:18.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:16:18.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:18.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_38 128M 2024-09-18T01:16:18.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:18.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:18.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:18.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:18.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:16:18.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:18.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:16:18.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:16:18.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:16:18.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:18.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_38 2024-09-18T01:16:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:16:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:16:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_38 snapshot 2024-09-18T01:16:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:16:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_38 snapshot 2024-09-18T01:16:19.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:16:19.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_38 2024-09-18T01:16:19.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_38': 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13d3239f7918 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13d3239f7918 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:18 2024 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:18 2024 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:18 2024 2024-09-18T01:16:19.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:16:19.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:16:19.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 4ff798bb-162f-4ad6-acae-69c5a62867d9 2024-09-18T01:16:19.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:16:19.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:16:19.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_38 2024-09-18T01:16:19.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:19.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:19.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:16:19.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:16:19.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=6 2024-09-18T01:16:19.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:16:19.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:16:19.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 25338880 bytes 2024-09-18T01:16:19.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 61024256 bytes 2024-09-18T01:16:19.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 76898304 bytes 2024-09-18T01:16:20.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 130965504 bytes 2024-09-18T01:16:20.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 136722432 bytes 2024-09-18T01:16:20.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 146039808 bytes 2024-09-18T01:16:20.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 188894208 bytes 2024-09-18T01:16:20.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 190332928 bytes 2024-09-18T01:16:20.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 229127168 bytes 2024-09-18T01:16:20.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 273693696 bytes 2024-09-18T01:16:20.919 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 313915392 bytes 2024-09-18T01:16:21.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 411363328 bytes 2024-09-18T01:16:21.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 494182400 bytes 2024-09-18T01:16:21.471 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 613359616 bytes 2024-09-18T01:16:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 691297280 bytes 2024-09-18T01:16:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 746142720 bytes 2024-09-18T01:16:21.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 789166080 bytes 2024-09-18T01:16:22.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 829390848 bytes 2024-09-18T01:16:22.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 884178944 bytes 2024-09-18T01:16:22.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 948298752 bytes 2024-09-18T01:16:22.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 985919488 bytes 2024-09-18T01:16:22.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1004069888 bytes 2024-09-18T01:16:22.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1037201408 bytes 2024-09-18T01:16:22.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1123300352 bytes 2024-09-18T01:16:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1125182464 bytes 2024-09-18T01:16:22.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1141296128 bytes 2024-09-18T01:16:23.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1156834304 bytes 2024-09-18T01:16:23.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1211873280 bytes 2024-09-18T01:16:23.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1286087680 bytes 2024-09-18T01:16:23.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1308823552 bytes 2024-09-18T01:16:23.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1316664320 bytes 2024-09-18T01:16:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1371036672 bytes 2024-09-18T01:16:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1405325312 bytes 2024-09-18T01:16:23.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1444410368 bytes 2024-09-18T01:16:24.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1481380864 bytes 2024-09-18T01:16:24.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1525579776 bytes 2024-09-18T01:16:24.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1594036224 bytes 2024-09-18T01:16:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1679367168 bytes 2024-09-18T01:16:24.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1767296000 bytes 2024-09-18T01:16:24.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1868433408 bytes 2024-09-18T01:16:25.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1940336640 bytes 2024-09-18T01:16:25.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 2014171136 bytes 2024-09-18T01:16:25.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2030957568 bytes 2024-09-18T01:16:25.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2070463488 bytes 2024-09-18T01:16:25.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:16:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:16:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:16:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:16:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_39 2024-09-18T01:16:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 snapshot 128M 2024-09-18T01:16:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:25.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:25.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:25.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:16:25.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:25.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:25.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:16:25.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:25.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_39 128M 2024-09-18T01:16:25.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:25.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:16:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:16:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:16:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:16:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:25.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_39 2024-09-18T01:16:25.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:16:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:16:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_39 snapshot 2024-09-18T01:16:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:16:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_39 snapshot 2024-09-18T01:16:26.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:16:26.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_39 2024-09-18T01:16:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_39': 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13dc894f10ba 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13dc894f10ba 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:25 2024 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:25 2024 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:25 2024 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:16:26.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:16:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 6f579a7e-81b1-4594-9e4f-741a03c86452 2024-09-18T01:16:26.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:16:26.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:16:26.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_39 2024-09-18T01:16:26.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:26.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:26.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:16:26.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:16:26.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=12 2024-09-18T01:16:26.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:16:26.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:16:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 39469056 bytes 2024-09-18T01:16:26.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 78590976 bytes 2024-09-18T01:16:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 143963136 bytes 2024-09-18T01:16:27.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 196237312 bytes 2024-09-18T01:16:27.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 286104576 bytes 2024-09-18T01:16:27.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 353552384 bytes 2024-09-18T01:16:27.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 395320320 bytes 2024-09-18T01:16:27.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 412187648 bytes 2024-09-18T01:16:27.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 412814336 bytes 2024-09-18T01:16:27.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 478356480 bytes 2024-09-18T01:16:28.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 537906176 bytes 2024-09-18T01:16:28.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 609092608 bytes 2024-09-18T01:16:28.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 679797760 bytes 2024-09-18T01:16:28.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 739763200 bytes 2024-09-18T01:16:28.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 800779264 bytes 2024-09-18T01:16:28.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 820743168 bytes 2024-09-18T01:16:28.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 869779456 bytes 2024-09-18T01:16:29.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 925476864 bytes 2024-09-18T01:16:29.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 975846400 bytes 2024-09-18T01:16:29.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 1005395968 bytes 2024-09-18T01:16:29.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1093996544 bytes 2024-09-18T01:16:29.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1185645568 bytes 2024-09-18T01:16:29.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1203788800 bytes 2024-09-18T01:16:29.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1252688896 bytes 2024-09-18T01:16:30.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1294251008 bytes 2024-09-18T01:16:30.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1339302912 bytes 2024-09-18T01:16:30.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1377938432 bytes 2024-09-18T01:16:30.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1401587712 bytes 2024-09-18T01:16:30.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1445702656 bytes 2024-09-18T01:16:30.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1501019136 bytes 2024-09-18T01:16:30.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1515578368 bytes 2024-09-18T01:16:30.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1594321920 bytes 2024-09-18T01:16:31.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1624632320 bytes 2024-09-18T01:16:31.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1640152064 bytes 2024-09-18T01:16:31.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1684884480 bytes 2024-09-18T01:16:31.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1707323392 bytes 2024-09-18T01:16:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1778570240 bytes 2024-09-18T01:16:31.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1868681216 bytes 2024-09-18T01:16:31.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1907732480 bytes 2024-09-18T01:16:31.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1941061632 bytes 2024-09-18T01:16:32.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1997008896 bytes 2024-09-18T01:16:32.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 2078958592 bytes 2024-09-18T01:16:32.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2086525952 bytes 2024-09-18T01:16:32.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2102037504 bytes 2024-09-18T01:16:32.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2122908672 bytes 2024-09-18T01:16:32.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2165188608 bytes 2024-09-18T01:16:32.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2199880704 bytes 2024-09-18T01:16:32.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2238389248 bytes 2024-09-18T01:16:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2298786816 bytes 2024-09-18T01:16:33.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2299424768 bytes 2024-09-18T01:16:33.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2340454400 bytes 2024-09-18T01:16:33.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2383632384 bytes 2024-09-18T01:16:33.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2391725056 bytes 2024-09-18T01:16:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2449049600 bytes 2024-09-18T01:16:33.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2465277952 bytes 2024-09-18T01:16:33.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2506935296 bytes 2024-09-18T01:16:34.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2529678336 bytes 2024-09-18T01:16:34.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2572653568 bytes 2024-09-18T01:16:34.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2643408896 bytes 2024-09-18T01:16:34.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2668362752 bytes 2024-09-18T01:16:34.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2748311552 bytes 2024-09-18T01:16:34.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2793854976 bytes 2024-09-18T01:16:34.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2869765120 bytes 2024-09-18T01:16:35.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2889827328 bytes 2024-09-18T01:16:35.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2933843968 bytes 2024-09-18T01:16:35.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2958569472 bytes 2024-09-18T01:16:35.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3003422720 bytes 2024-09-18T01:16:35.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3065573376 bytes 2024-09-18T01:16:35.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3133673472 bytes 2024-09-18T01:16:35.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3134262272 bytes 2024-09-18T01:16:35.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3174925312 bytes 2024-09-18T01:16:36.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3213090816 bytes 2024-09-18T01:16:36.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3234974720 bytes 2024-09-18T01:16:36.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3263834112 bytes 2024-09-18T01:16:36.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3321197568 bytes 2024-09-18T01:16:36.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3341236224 bytes 2024-09-18T01:16:36.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3361761280 bytes 2024-09-18T01:16:36.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3463256064 bytes 2024-09-18T01:16:36.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3517489152 bytes 2024-09-18T01:16:37.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3557654528 bytes 2024-09-18T01:16:37.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3579442176 bytes 2024-09-18T01:16:37.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3620996096 bytes 2024-09-18T01:16:37.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3661332480 bytes 2024-09-18T01:16:37.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3693095936 bytes 2024-09-18T01:16:37.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3714937856 bytes 2024-09-18T01:16:37.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3734914048 bytes 2024-09-18T01:16:37.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3784042496 bytes 2024-09-18T01:16:37.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3818573824 bytes 2024-09-18T01:16:38.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3881328640 bytes 2024-09-18T01:16:38.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3927128064 bytes 2024-09-18T01:16:38.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3945575424 bytes 2024-09-18T01:16:38.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3970329600 bytes 2024-09-18T01:16:38.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4014472192 bytes 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_40 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 snapshot 128M 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:38.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:16:38.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:38.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:38.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:16:38.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:38.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_40 128M 2024-09-18T01:16:38.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:38.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:38.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:38.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_40 2024-09-18T01:16:38.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:16:38.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:38.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:16:38.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:38.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_40 snapshot 2024-09-18T01:16:38.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:38.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:38.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:16:38.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:38.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_40 snapshot 2024-09-18T01:16:39.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:16:39.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_40 2024-09-18T01:16:39.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_40': 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13e5c65edae9 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13e5c65edae9 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:38 2024 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:38 2024 2024-09-18T01:16:39.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:38 2024 2024-09-18T01:16:39.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:16:39.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:16:39.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 75569094-ed91-4db4-95b0-2512a6bdcc51 2024-09-18T01:16:39.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:16:39.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:16:39.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_40 2024-09-18T01:16:39.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:39.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:39.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:16:39.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:16:39.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=9 2024-09-18T01:16:39.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:16:39.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:16:39.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 27954176 bytes 2024-09-18T01:16:39.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 82631680 bytes 2024-09-18T01:16:40.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 107246592 bytes 2024-09-18T01:16:40.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 207400960 bytes 2024-09-18T01:16:40.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 271175680 bytes 2024-09-18T01:16:40.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 296273920 bytes 2024-09-18T01:16:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 321371136 bytes 2024-09-18T01:16:40.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 374045696 bytes 2024-09-18T01:16:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 454731776 bytes 2024-09-18T01:16:41.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 571541504 bytes 2024-09-18T01:16:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 615838720 bytes 2024-09-18T01:16:41.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 695031808 bytes 2024-09-18T01:16:41.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 741317632 bytes 2024-09-18T01:16:41.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 790121472 bytes 2024-09-18T01:16:41.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 812340224 bytes 2024-09-18T01:16:41.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 838016000 bytes 2024-09-18T01:16:41.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 851038208 bytes 2024-09-18T01:16:42.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 883772416 bytes 2024-09-18T01:16:42.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 923141120 bytes 2024-09-18T01:16:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 963146752 bytes 2024-09-18T01:16:42.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 996033536 bytes 2024-09-18T01:16:42.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1042829312 bytes 2024-09-18T01:16:42.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1068660736 bytes 2024-09-18T01:16:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1138057216 bytes 2024-09-18T01:16:42.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1191963648 bytes 2024-09-18T01:16:43.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1277598720 bytes 2024-09-18T01:16:43.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1310480384 bytes 2024-09-18T01:16:43.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1311070208 bytes 2024-09-18T01:16:43.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1312747520 bytes 2024-09-18T01:16:43.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1373444096 bytes 2024-09-18T01:16:43.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1411517440 bytes 2024-09-18T01:16:43.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1436805120 bytes 2024-09-18T01:16:44.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1518595072 bytes 2024-09-18T01:16:44.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1536052224 bytes 2024-09-18T01:16:44.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1560920064 bytes 2024-09-18T01:16:44.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1582532608 bytes 2024-09-18T01:16:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1605577728 bytes 2024-09-18T01:16:44.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1641340928 bytes 2024-09-18T01:16:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1697861632 bytes 2024-09-18T01:16:44.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1721591808 bytes 2024-09-18T01:16:44.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1732758528 bytes 2024-09-18T01:16:45.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1751127040 bytes 2024-09-18T01:16:45.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1827653632 bytes 2024-09-18T01:16:45.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1845581824 bytes 2024-09-18T01:16:45.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1880573952 bytes 2024-09-18T01:16:45.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1897120768 bytes 2024-09-18T01:16:45.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1909224448 bytes 2024-09-18T01:16:45.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1910709248 bytes 2024-09-18T01:16:45.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1938219008 bytes 2024-09-18T01:16:45.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1982572544 bytes 2024-09-18T01:16:46.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2012328960 bytes 2024-09-18T01:16:46.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2060435456 bytes 2024-09-18T01:16:46.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2088143872 bytes 2024-09-18T01:16:46.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2120144896 bytes 2024-09-18T01:16:46.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2126323712 bytes 2024-09-18T01:16:46.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2146861056 bytes 2024-09-18T01:16:46.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2163009536 bytes 2024-09-18T01:16:46.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2205734912 bytes 2024-09-18T01:16:47.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2326211584 bytes 2024-09-18T01:16:47.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2351899648 bytes 2024-09-18T01:16:47.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2387758080 bytes 2024-09-18T01:16:47.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2421551104 bytes 2024-09-18T01:16:47.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2468374528 bytes 2024-09-18T01:16:47.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2493857792 bytes 2024-09-18T01:16:47.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2527381504 bytes 2024-09-18T01:16:47.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2564608000 bytes 2024-09-18T01:16:48.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2596581376 bytes 2024-09-18T01:16:48.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2597166080 bytes 2024-09-18T01:16:48.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2652696576 bytes 2024-09-18T01:16:48.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2679416832 bytes 2024-09-18T01:16:48.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2687935488 bytes 2024-09-18T01:16:48.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:16:48.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:16:48.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:16:48.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:48.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:16:48.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_41 2024-09-18T01:16:48.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 snapshot 128M 2024-09-18T01:16:48.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:48.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:16:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:16:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:48.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_41 128M 2024-09-18T01:16:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:16:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:16:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:16:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:16:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:48.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_41 2024-09-18T01:16:48.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:16:48.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:48.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:16:48.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:48.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_41 snapshot 2024-09-18T01:16:48.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:16:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:48.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_41 snapshot 2024-09-18T01:16:49.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:16:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_41 2024-09-18T01:16:49.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_41': 2024-09-18T01:16:49.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:16:49.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:16:49.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:16:49.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13ed5457c2d6 2024-09-18T01:16:49.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13ed5457c2d6 2024-09-18T01:16:49.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:16:49.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:16:49.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:16:49.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:16:49.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:48 2024 2024-09-18T01:16:49.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:48 2024 2024-09-18T01:16:49.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:48 2024 2024-09-18T01:16:49.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:16:49.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:16:49.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 0071b191-d397-4a11-b33d-d10f4477a5a3 2024-09-18T01:16:49.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:16:49.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:16:49.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_41 2024-09-18T01:16:49.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:49.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:49.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:16:49.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:16:49.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=7 2024-09-18T01:16:49.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:16:49.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:16:49.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 120474624 bytes 2024-09-18T01:16:50.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 164143104 bytes 2024-09-18T01:16:50.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 186766336 bytes 2024-09-18T01:16:50.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 222032896 bytes 2024-09-18T01:16:50.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 261779456 bytes 2024-09-18T01:16:50.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 262428672 bytes 2024-09-18T01:16:50.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 270705664 bytes 2024-09-18T01:16:50.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 296494080 bytes 2024-09-18T01:16:50.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 332289024 bytes 2024-09-18T01:16:50.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 377712640 bytes 2024-09-18T01:16:51.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 408493056 bytes 2024-09-18T01:16:51.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 435322880 bytes 2024-09-18T01:16:51.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 467301376 bytes 2024-09-18T01:16:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 556455936 bytes 2024-09-18T01:16:51.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 584070144 bytes 2024-09-18T01:16:51.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 629691392 bytes 2024-09-18T01:16:51.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 678852608 bytes 2024-09-18T01:16:51.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 712931328 bytes 2024-09-18T01:16:52.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 761629696 bytes 2024-09-18T01:16:52.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 839749632 bytes 2024-09-18T01:16:52.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 872536064 bytes 2024-09-18T01:16:52.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 936132608 bytes 2024-09-18T01:16:52.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1021000704 bytes 2024-09-18T01:16:52.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1043530752 bytes 2024-09-18T01:16:52.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1061731328 bytes 2024-09-18T01:16:53.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1118291968 bytes 2024-09-18T01:16:53.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1128964096 bytes 2024-09-18T01:16:53.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1167545344 bytes 2024-09-18T01:16:53.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1259337728 bytes 2024-09-18T01:16:53.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1329286144 bytes 2024-09-18T01:16:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1354432512 bytes 2024-09-18T01:16:53.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1409929216 bytes 2024-09-18T01:16:54.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1456214016 bytes 2024-09-18T01:16:54.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1549062144 bytes 2024-09-18T01:16:54.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1564493824 bytes 2024-09-18T01:16:54.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1593696256 bytes 2024-09-18T01:16:54.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1637283840 bytes 2024-09-18T01:16:54.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1720528896 bytes 2024-09-18T01:16:54.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1758870528 bytes 2024-09-18T01:16:55.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1804099584 bytes 2024-09-18T01:16:55.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1836695552 bytes 2024-09-18T01:16:55.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1953033216 bytes 2024-09-18T01:16:55.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2081558528 bytes 2024-09-18T01:16:55.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2121440256 bytes 2024-09-18T01:16:55.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2144016384 bytes 2024-09-18T01:16:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2168518656 bytes 2024-09-18T01:16:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2185858048 bytes 2024-09-18T01:16:56.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2258792448 bytes 2024-09-18T01:16:56.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2303950848 bytes 2024-09-18T01:16:56.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2387845120 bytes 2024-09-18T01:16:56.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2414956544 bytes 2024-09-18T01:16:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:16:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:16:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:16:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:16:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_42 2024-09-18T01:16:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 snapshot 128M 2024-09-18T01:16:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:56.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:16:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:16:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_42 128M 2024-09-18T01:16:56.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:16:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:16:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:16:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:16:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:16:56.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_42 2024-09-18T01:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_42 snapshot 2024-09-18T01:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_42 snapshot 2024-09-18T01:16:57.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:16:57.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_42 2024-09-18T01:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_42': 2024-09-18T01:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:16:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13f6ad6d589f 2024-09-18T01:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13f6ad6d589f 2024-09-18T01:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:16:57.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:56 2024 2024-09-18T01:16:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:56 2024 2024-09-18T01:16:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:56 2024 2024-09-18T01:16:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:16:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:16:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: aeaeb058-01f1-4690-999c-8c21dad1561e 2024-09-18T01:16:57.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:16:57.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:16:57.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_42 2024-09-18T01:16:57.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:16:57.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:16:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:16:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:16:57.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=13 2024-09-18T01:16:57.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:16:57.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:16:57.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 27448320 bytes 2024-09-18T01:16:57.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 28057600 bytes 2024-09-18T01:16:58.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 68977664 bytes 2024-09-18T01:16:58.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 89414656 bytes 2024-09-18T01:16:58.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 98765824 bytes 2024-09-18T01:16:58.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 136046592 bytes 2024-09-18T01:16:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 150724608 bytes 2024-09-18T01:16:58.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 157265920 bytes 2024-09-18T01:16:58.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 206813184 bytes 2024-09-18T01:16:58.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 246334464 bytes 2024-09-18T01:16:59.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 331014144 bytes 2024-09-18T01:16:59.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 366531584 bytes 2024-09-18T01:16:59.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 400023552 bytes 2024-09-18T01:16:59.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 465998848 bytes 2024-09-18T01:16:59.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 536408064 bytes 2024-09-18T01:16:59.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 552384512 bytes 2024-09-18T01:16:59.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 585220096 bytes 2024-09-18T01:16:59.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 585831424 bytes 2024-09-18T01:17:00.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 670237696 bytes 2024-09-18T01:17:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 705979392 bytes 2024-09-18T01:17:00.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 747523072 bytes 2024-09-18T01:17:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 787952640 bytes 2024-09-18T01:17:00.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 814447616 bytes 2024-09-18T01:17:00.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 908771328 bytes 2024-09-18T01:17:00.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 963089408 bytes 2024-09-18T01:17:01.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1037263872 bytes 2024-09-18T01:17:01.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1053664256 bytes 2024-09-18T01:17:01.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1054258176 bytes 2024-09-18T01:17:01.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1062870016 bytes 2024-09-18T01:17:01.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1105339392 bytes 2024-09-18T01:17:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1105925120 bytes 2024-09-18T01:17:01.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1146255360 bytes 2024-09-18T01:17:01.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1171186688 bytes 2024-09-18T01:17:02.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1214151680 bytes 2024-09-18T01:17:02.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1270775808 bytes 2024-09-18T01:17:02.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1332333568 bytes 2024-09-18T01:17:02.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1398238208 bytes 2024-09-18T01:17:02.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1446117376 bytes 2024-09-18T01:17:02.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1498158080 bytes 2024-09-18T01:17:02.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1516114944 bytes 2024-09-18T01:17:03.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1626486784 bytes 2024-09-18T01:17:03.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1704067072 bytes 2024-09-18T01:17:03.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1732256768 bytes 2024-09-18T01:17:03.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1759854592 bytes 2024-09-18T01:17:03.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1850823680 bytes 2024-09-18T01:17:03.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1893328896 bytes 2024-09-18T01:17:03.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1917708288 bytes 2024-09-18T01:17:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1960409088 bytes 2024-09-18T01:17:04.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1995089920 bytes 2024-09-18T01:17:04.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2010438656 bytes 2024-09-18T01:17:04.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2105159680 bytes 2024-09-18T01:17:04.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2175230976 bytes 2024-09-18T01:17:04.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2274003968 bytes 2024-09-18T01:17:04.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2355612672 bytes 2024-09-18T01:17:05.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2421639168 bytes 2024-09-18T01:17:05.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2509151232 bytes 2024-09-18T01:17:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2556686336 bytes 2024-09-18T01:17:05.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2617549824 bytes 2024-09-18T01:17:05.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2618189824 bytes 2024-09-18T01:17:05.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2679319552 bytes 2024-09-18T01:17:05.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2681132032 bytes 2024-09-18T01:17:05.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2707948544 bytes 2024-09-18T01:17:06.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2751960064 bytes 2024-09-18T01:17:06.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2789059584 bytes 2024-09-18T01:17:06.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2847959040 bytes 2024-09-18T01:17:06.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2889994240 bytes 2024-09-18T01:17:06.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2978931712 bytes 2024-09-18T01:17:06.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2988961792 bytes 2024-09-18T01:17:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3007826944 bytes 2024-09-18T01:17:06.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3073203200 bytes 2024-09-18T01:17:07.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3113504768 bytes 2024-09-18T01:17:07.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3146416128 bytes 2024-09-18T01:17:07.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3232972800 bytes 2024-09-18T01:17:07.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3249115136 bytes 2024-09-18T01:17:07.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3256576000 bytes 2024-09-18T01:17:07.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3329274880 bytes 2024-09-18T01:17:07.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3387234304 bytes 2024-09-18T01:17:08.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3429603328 bytes 2024-09-18T01:17:08.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3511169024 bytes 2024-09-18T01:17:08.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3574713344 bytes 2024-09-18T01:17:08.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3606308864 bytes 2024-09-18T01:17:08.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3606914048 bytes 2024-09-18T01:17:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3632649216 bytes 2024-09-18T01:17:08.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3748153344 bytes 2024-09-18T01:17:08.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3763039232 bytes 2024-09-18T01:17:09.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3834299392 bytes 2024-09-18T01:17:09.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3855919104 bytes 2024-09-18T01:17:09.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3941662720 bytes 2024-09-18T01:17:09.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3955771392 bytes 2024-09-18T01:17:09.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3994983424 bytes 2024-09-18T01:17:09.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 4068742144 bytes 2024-09-18T01:17:09.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4108882944 bytes 2024-09-18T01:17:10.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4109517824 bytes 2024-09-18T01:17:10.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4121217024 bytes 2024-09-18T01:17:10.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4163558400 bytes 2024-09-18T01:17:10.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4206701568 bytes 2024-09-18T01:17:10.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4255380480 bytes 2024-09-18T01:17:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4288055296 bytes 2024-09-18T01:17:10.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:17:10.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:17:10.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:17:10.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:17:10.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:17:10.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_43 2024-09-18T01:17:10.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 snapshot 128M 2024-09-18T01:17:10.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:10.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:10.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:17:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:17:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:17:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_43 128M 2024-09-18T01:17:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:10.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:17:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:17:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:17:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:17:10.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:10.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_43 2024-09-18T01:17:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:17:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:17:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:17:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_43 snapshot 2024-09-18T01:17:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:17:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:17:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_43 snapshot 2024-09-18T01:17:11.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:17:11.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_43 2024-09-18T01:17:11.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_43': 2024-09-18T01:17:11.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:17:11.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:17:11.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:17:11.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 13feca6dcdcc 2024-09-18T01:17:11.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.13feca6dcdcc 2024-09-18T01:17:11.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:17:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:17:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:17:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:17:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:17:10 2024 2024-09-18T01:17:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:17:10 2024 2024-09-18T01:17:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:17:10 2024 2024-09-18T01:17:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:17:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:17:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: bf14a2b6-1625-4fba-8a13-3e69c7467ad0 2024-09-18T01:17:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:17:11.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:17:11.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_43 2024-09-18T01:17:11.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:11.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:11.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:17:11.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:17:11.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=13 2024-09-18T01:17:11.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:17:11.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:17:11.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 44125184 bytes 2024-09-18T01:17:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 94895104 bytes 2024-09-18T01:17:12.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 190823424 bytes 2024-09-18T01:17:12.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 244710400 bytes 2024-09-18T01:17:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 300247040 bytes 2024-09-18T01:17:12.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 338571264 bytes 2024-09-18T01:17:12.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 399462400 bytes 2024-09-18T01:17:12.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 430786560 bytes 2024-09-18T01:17:12.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 462803968 bytes 2024-09-18T01:17:13.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 533697536 bytes 2024-09-18T01:17:13.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 540473344 bytes 2024-09-18T01:17:13.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 564559872 bytes 2024-09-18T01:17:13.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 623982592 bytes 2024-09-18T01:17:13.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 697870336 bytes 2024-09-18T01:17:13.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 746187776 bytes 2024-09-18T01:17:13.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 801421312 bytes 2024-09-18T01:17:14.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 817732608 bytes 2024-09-18T01:17:14.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 860560384 bytes 2024-09-18T01:17:14.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 934300672 bytes 2024-09-18T01:17:14.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 996444160 bytes 2024-09-18T01:17:14.440 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-18T01:17:14.441 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-18T01:17:14.441 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-18T01:17:14.445 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:17:14.443+0000 7f2dee9ec640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 48576) UID: 1000 2024-09-18T01:17:14.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1025763328 bytes 2024-09-18T01:17:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1050212352 bytes 2024-09-18T01:17:14.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1116393472 bytes 2024-09-18T01:17:15.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1213977600 bytes 2024-09-18T01:17:15.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1258546176 bytes 2024-09-18T01:17:15.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1283756032 bytes 2024-09-18T01:17:15.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1343507456 bytes 2024-09-18T01:17:15.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1387838464 bytes 2024-09-18T01:17:15.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1391283200 bytes 2024-09-18T01:17:15.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1450229760 bytes 2024-09-18T01:17:15.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1509736448 bytes 2024-09-18T01:17:16.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1533432832 bytes 2024-09-18T01:17:16.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1597166592 bytes 2024-09-18T01:17:16.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1615756288 bytes 2024-09-18T01:17:16.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1640022016 bytes 2024-09-18T01:17:16.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1680036864 bytes 2024-09-18T01:17:16.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1752757248 bytes 2024-09-18T01:17:16.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1786263552 bytes 2024-09-18T01:17:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1825590272 bytes 2024-09-18T01:17:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1884042240 bytes 2024-09-18T01:17:17.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1930697728 bytes 2024-09-18T01:17:17.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1951783936 bytes 2024-09-18T01:17:17.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2009528320 bytes 2024-09-18T01:17:17.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2034779136 bytes 2024-09-18T01:17:17.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2091511808 bytes 2024-09-18T01:17:17.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2141938688 bytes 2024-09-18T01:17:17.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2173317120 bytes 2024-09-18T01:17:18.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2237333504 bytes 2024-09-18T01:17:18.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2321576960 bytes 2024-09-18T01:17:18.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2381641728 bytes 2024-09-18T01:17:18.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2425377792 bytes 2024-09-18T01:17:18.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2489021440 bytes 2024-09-18T01:17:18.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2513274880 bytes 2024-09-18T01:17:18.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2528114688 bytes 2024-09-18T01:17:19.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2553330688 bytes 2024-09-18T01:17:19.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2610564096 bytes 2024-09-18T01:17:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2611159040 bytes 2024-09-18T01:17:19.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2651163648 bytes 2024-09-18T01:17:19.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2697264128 bytes 2024-09-18T01:17:19.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2796149760 bytes 2024-09-18T01:17:19.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2832932864 bytes 2024-09-18T01:17:19.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2868582400 bytes 2024-09-18T01:17:20.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2881711104 bytes 2024-09-18T01:17:20.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2922448896 bytes 2024-09-18T01:17:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 3000060928 bytes 2024-09-18T01:17:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 3016868864 bytes 2024-09-18T01:17:20.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3062812672 bytes 2024-09-18T01:17:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3134377984 bytes 2024-09-18T01:17:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3191848960 bytes 2024-09-18T01:17:21.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3275361280 bytes 2024-09-18T01:17:21.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3283888128 bytes 2024-09-18T01:17:21.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3314561024 bytes 2024-09-18T01:17:21.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3344508928 bytes 2024-09-18T01:17:21.442 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-18T01:17:21.442 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:17:21.442 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-18T01:17:21.443 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-18T01:17:21.443 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-18T01:17:21.446 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-18T01:17:21.446 INFO:tasks.rbd_mirror_thrash:waiting for 107 secs before thrashing 2024-09-18T01:17:21.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3421234176 bytes 2024-09-18T01:17:21.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3423355904 bytes 2024-09-18T01:17:21.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3475550208 bytes 2024-09-18T01:17:21.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3510063104 bytes 2024-09-18T01:17:22.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3510673408 bytes 2024-09-18T01:17:22.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3538367488 bytes 2024-09-18T01:17:22.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3643996160 bytes 2024-09-18T01:17:22.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3672493056 bytes 2024-09-18T01:17:22.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3736326144 bytes 2024-09-18T01:17:22.783 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3770733568 bytes 2024-09-18T01:17:22.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3824039936 bytes 2024-09-18T01:17:23.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3854939136 bytes 2024-09-18T01:17:23.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3865549824 bytes 2024-09-18T01:17:23.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3914463232 bytes 2024-09-18T01:17:23.406 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3954718720 bytes 2024-09-18T01:17:23.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 4015032320 bytes 2024-09-18T01:17:23.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 4090808320 bytes 2024-09-18T01:17:23.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 4128312320 bytes 2024-09-18T01:17:23.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4163394560 bytes 2024-09-18T01:17:24.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4239767552 bytes 2024-09-18T01:17:24.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4313353216 bytes 2024-09-18T01:17:24.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4317796352 bytes 2024-09-18T01:17:24.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4371262464 bytes 2024-09-18T01:17:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:17:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:17:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:17:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:17:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:17:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_44 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 snapshot 128M 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_44 128M 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:24.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:24.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:17:24.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:24.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:17:24.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:17:24.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:17:24.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:24.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_44 2024-09-18T01:17:24.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:17:24.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:24.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:17:24.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:17:24.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_44 snapshot 2024-09-18T01:17:24.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:24.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:24.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:17:24.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:17:24.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_44 snapshot 2024-09-18T01:17:25.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:17:25.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_44 2024-09-18T01:17:25.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_44': 2024-09-18T01:17:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:17:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:17:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:17:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 140ccb4d43dd 2024-09-18T01:17:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.140ccb4d43dd 2024-09-18T01:17:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:17:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:17:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:17:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:17:25.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:17:24 2024 2024-09-18T01:17:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:17:24 2024 2024-09-18T01:17:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:17:24 2024 2024-09-18T01:17:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:17:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:17:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: f0afae9f-39f5-4655-a91e-7d6dfba2149c 2024-09-18T01:17:25.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:17:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:17:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_44 2024-09-18T01:17:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:17:25.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:17:25.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=8 2024-09-18T01:17:25.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:17:25.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:17:25.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 18396160 bytes 2024-09-18T01:17:25.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 67695616 bytes 2024-09-18T01:17:26.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 133365760 bytes 2024-09-18T01:17:26.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 176004096 bytes 2024-09-18T01:17:26.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 183075840 bytes 2024-09-18T01:17:26.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 203575296 bytes 2024-09-18T01:17:26.582 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 242023424 bytes 2024-09-18T01:17:26.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 307185664 bytes 2024-09-18T01:17:26.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 323723264 bytes 2024-09-18T01:17:26.939 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 339927040 bytes 2024-09-18T01:17:27.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 354302976 bytes 2024-09-18T01:17:27.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 483720192 bytes 2024-09-18T01:17:27.396 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 550438912 bytes 2024-09-18T01:17:27.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 553785344 bytes 2024-09-18T01:17:27.509 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.31574076444409194) > weight (0.25) 2024-09-18T01:17:27.509 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-18T01:17:27.509 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-18T01:17:27.509 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-09-18T01:17:27.510 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:17:27.508+0000 7fa2cdbde640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 46677) UID: 1000 2024-09-18T01:17:27.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 609160192 bytes 2024-09-18T01:17:27.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 612608000 bytes 2024-09-18T01:17:27.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 620870656 bytes 2024-09-18T01:17:27.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 638477312 bytes 2024-09-18T01:17:28.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 706236416 bytes 2024-09-18T01:17:28.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 743238656 bytes 2024-09-18T01:17:28.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 830527488 bytes 2024-09-18T01:17:28.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 854614016 bytes 2024-09-18T01:17:28.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 889233408 bytes 2024-09-18T01:17:28.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 904627200 bytes 2024-09-18T01:17:28.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 915590144 bytes 2024-09-18T01:17:29.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 975264768 bytes 2024-09-18T01:17:29.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 983709696 bytes 2024-09-18T01:17:29.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1038455808 bytes 2024-09-18T01:17:29.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1054887936 bytes 2024-09-18T01:17:29.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1129077760 bytes 2024-09-18T01:17:29.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1167949824 bytes 2024-09-18T01:17:29.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1173693440 bytes 2024-09-18T01:17:29.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1195240448 bytes 2024-09-18T01:17:29.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1212437504 bytes 2024-09-18T01:17:30.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1238461440 bytes 2024-09-18T01:17:30.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1257039872 bytes 2024-09-18T01:17:30.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1307364352 bytes 2024-09-18T01:17:30.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1332018176 bytes 2024-09-18T01:17:30.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1337675776 bytes 2024-09-18T01:17:30.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1351859200 bytes 2024-09-18T01:17:30.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1373256704 bytes 2024-09-18T01:17:30.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1380965376 bytes 2024-09-18T01:17:30.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1433832448 bytes 2024-09-18T01:17:31.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1496259584 bytes 2024-09-18T01:17:31.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1496890368 bytes 2024-09-18T01:17:31.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1532220416 bytes 2024-09-18T01:17:31.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1553110016 bytes 2024-09-18T01:17:31.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1616068608 bytes 2024-09-18T01:17:31.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1637993472 bytes 2024-09-18T01:17:31.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1682231296 bytes 2024-09-18T01:17:31.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 1706467328 bytes 2024-09-18T01:17:32.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 1765062656 bytes 2024-09-18T01:17:32.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 1822182400 bytes 2024-09-18T01:17:32.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 1951669248 bytes 2024-09-18T01:17:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 1979746304 bytes 2024-09-18T01:17:32.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2004423680 bytes 2024-09-18T01:17:32.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2064125952 bytes 2024-09-18T01:17:33.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2105271296 bytes 2024-09-18T01:17:33.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2139218944 bytes 2024-09-18T01:17:33.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2182394880 bytes 2024-09-18T01:17:33.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2231976960 bytes 2024-09-18T01:17:33.510 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-18T01:17:33.510 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:17:33.510 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-18T01:17:33.510 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-18T01:17:33.511 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-18T01:17:33.513 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-18T01:17:33.514 INFO:tasks.rbd_mirror_thrash:waiting for 111 secs before thrashing 2024-09-18T01:17:33.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2282214400 bytes 2024-09-18T01:17:33.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:17:33.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:17:33.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:17:33.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:17:33.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:17:33.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_45 2024-09-18T01:17:33.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 snapshot 128M 2024-09-18T01:17:33.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:33.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:33.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:33.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:33.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:17:33.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:33.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:17:33.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:17:33.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_45 128M 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:33.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_45 2024-09-18T01:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:17:33.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_45 snapshot 2024-09-18T01:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:17:33.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_45 snapshot 2024-09-18T01:17:34.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:17:34.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_45 2024-09-18T01:17:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_45': 2024-09-18T01:17:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:17:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:17:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:17:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 141cb888402b 2024-09-18T01:17:34.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.141cb888402b 2024-09-18T01:17:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:17:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:17:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:17:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:17:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:17:33 2024 2024-09-18T01:17:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:17:33 2024 2024-09-18T01:17:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:17:33 2024 2024-09-18T01:17:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:17:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:17:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 4858f75f-8426-4450-aea4-9c743ab91e23 2024-09-18T01:17:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:17:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:17:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_45 2024-09-18T01:17:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:34.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:34.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:17:34.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:17:34.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=8 2024-09-18T01:17:34.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:17:34.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:17:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 43632640 bytes 2024-09-18T01:17:35.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 73940992 bytes 2024-09-18T01:17:35.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 131838976 bytes 2024-09-18T01:17:35.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 209007616 bytes 2024-09-18T01:17:35.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 222034944 bytes 2024-09-18T01:17:35.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 236300288 bytes 2024-09-18T01:17:35.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 299272192 bytes 2024-09-18T01:17:35.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 362045440 bytes 2024-09-18T01:17:35.919 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 382097408 bytes 2024-09-18T01:17:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 448008192 bytes 2024-09-18T01:17:36.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 517850112 bytes 2024-09-18T01:17:36.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 553182208 bytes 2024-09-18T01:17:36.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 595180544 bytes 2024-09-18T01:17:36.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 660452352 bytes 2024-09-18T01:17:36.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 693822464 bytes 2024-09-18T01:17:36.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 743110656 bytes 2024-09-18T01:17:36.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 787463168 bytes 2024-09-18T01:17:37.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 823542784 bytes 2024-09-18T01:17:37.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 864168960 bytes 2024-09-18T01:17:37.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 882277376 bytes 2024-09-18T01:17:37.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 950828032 bytes 2024-09-18T01:17:37.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 969959424 bytes 2024-09-18T01:17:37.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1008752640 bytes 2024-09-18T01:17:37.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1036200960 bytes 2024-09-18T01:17:37.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1072770048 bytes 2024-09-18T01:17:38.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1126116352 bytes 2024-09-18T01:17:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1177446400 bytes 2024-09-18T01:17:38.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1301742592 bytes 2024-09-18T01:17:38.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1367694336 bytes 2024-09-18T01:17:38.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1439417344 bytes 2024-09-18T01:17:38.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1510532096 bytes 2024-09-18T01:17:39.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1525102592 bytes 2024-09-18T01:17:39.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1590970368 bytes 2024-09-18T01:17:39.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1602935808 bytes 2024-09-18T01:17:39.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1655172096 bytes 2024-09-18T01:17:39.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1675203584 bytes 2024-09-18T01:17:39.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1751759872 bytes 2024-09-18T01:17:39.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1831416832 bytes 2024-09-18T01:17:39.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1888005120 bytes 2024-09-18T01:17:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1967054848 bytes 2024-09-18T01:17:40.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 2009773056 bytes 2024-09-18T01:17:40.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 2075739136 bytes 2024-09-18T01:17:40.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2119067648 bytes 2024-09-18T01:17:40.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2138469376 bytes 2024-09-18T01:17:40.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2210845696 bytes 2024-09-18T01:17:40.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2275438592 bytes 2024-09-18T01:17:41.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2305778688 bytes 2024-09-18T01:17:41.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2354269184 bytes 2024-09-18T01:17:41.363 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2414301184 bytes 2024-09-18T01:17:41.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2423142400 bytes 2024-09-18T01:17:41.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2453990400 bytes 2024-09-18T01:17:41.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2458032128 bytes 2024-09-18T01:17:41.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2489533440 bytes 2024-09-18T01:17:41.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2512290816 bytes 2024-09-18T01:17:41.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2531186688 bytes 2024-09-18T01:17:42.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2592441344 bytes 2024-09-18T01:17:42.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2635265024 bytes 2024-09-18T01:17:42.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2692460544 bytes 2024-09-18T01:17:42.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2765709312 bytes 2024-09-18T01:17:42.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2825873408 bytes 2024-09-18T01:17:42.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:17:42.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:17:42.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:17:42.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:17:42.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:17:42.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_46 2024-09-18T01:17:42.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 snapshot 128M 2024-09-18T01:17:42.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:42.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:42.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:42.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_46 128M 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:17:42.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:17:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:17:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:17:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:42.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_46 2024-09-18T01:17:42.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:17:42.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:42.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:17:42.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:17:42.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_46 snapshot 2024-09-18T01:17:42.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:42.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:42.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:17:42.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:17:42.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_46 snapshot 2024-09-18T01:17:42.940 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:17:42.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_46 2024-09-18T01:17:42.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_46': 2024-09-18T01:17:42.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:17:42.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:17:42.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:17:42.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 142b39616a6e 2024-09-18T01:17:42.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.142b39616a6e 2024-09-18T01:17:42.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:17:42.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:17:42.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:17:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:17:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:17:42 2024 2024-09-18T01:17:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:17:42 2024 2024-09-18T01:17:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:17:42 2024 2024-09-18T01:17:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:17:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:17:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 48442b3f-58cc-4f8f-8a64-4a412ad18406 2024-09-18T01:17:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:17:42.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:17:42.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_46 2024-09-18T01:17:42.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:42.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:42.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:17:42.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:17:42.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:17:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 61880320 bytes 2024-09-18T01:17:43.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 100864000 bytes 2024-09-18T01:17:43.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 131864576 bytes 2024-09-18T01:17:43.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 178490368 bytes 2024-09-18T01:17:43.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 204272640 bytes 2024-09-18T01:17:43.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 256920576 bytes 2024-09-18T01:17:43.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 261692416 bytes 2024-09-18T01:17:44.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 349056000 bytes 2024-09-18T01:17:44.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 349662208 bytes 2024-09-18T01:17:44.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 399847424 bytes 2024-09-18T01:17:44.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 471397376 bytes 2024-09-18T01:17:44.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 528500736 bytes 2024-09-18T01:17:44.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 551064576 bytes 2024-09-18T01:17:44.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 594523136 bytes 2024-09-18T01:17:44.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 624275456 bytes 2024-09-18T01:17:45.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 684304384 bytes 2024-09-18T01:17:45.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 719668224 bytes 2024-09-18T01:17:45.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 741766144 bytes 2024-09-18T01:17:45.423 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 787650560 bytes 2024-09-18T01:17:45.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 855543808 bytes 2024-09-18T01:17:45.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 876987392 bytes 2024-09-18T01:17:45.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 935054336 bytes 2024-09-18T01:17:45.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 966473728 bytes 2024-09-18T01:17:46.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1008011264 bytes 2024-09-18T01:17:46.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1052311552 bytes 2024-09-18T01:17:46.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1088146432 bytes 2024-09-18T01:17:46.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1103924224 bytes 2024-09-18T01:17:46.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1134329856 bytes 2024-09-18T01:17:46.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1242923008 bytes 2024-09-18T01:17:46.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1283040256 bytes 2024-09-18T01:17:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1348933632 bytes 2024-09-18T01:17:47.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1398821888 bytes 2024-09-18T01:17:47.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1422923776 bytes 2024-09-18T01:17:47.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1481084928 bytes 2024-09-18T01:17:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1516880896 bytes 2024-09-18T01:17:47.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1525934080 bytes 2024-09-18T01:17:47.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1549531136 bytes 2024-09-18T01:17:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1594343424 bytes 2024-09-18T01:17:48.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1648183296 bytes 2024-09-18T01:17:48.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1706644480 bytes 2024-09-18T01:17:48.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1756721152 bytes 2024-09-18T01:17:48.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1804028928 bytes 2024-09-18T01:17:48.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1890760704 bytes 2024-09-18T01:17:48.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1916932096 bytes 2024-09-18T01:17:48.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1951875072 bytes 2024-09-18T01:17:48.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2004186112 bytes 2024-09-18T01:17:49.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2025006080 bytes 2024-09-18T01:17:49.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2025633792 bytes 2024-09-18T01:17:49.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2091005952 bytes 2024-09-18T01:17:49.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2180762624 bytes 2024-09-18T01:17:49.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2295533568 bytes 2024-09-18T01:17:49.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2304382976 bytes 2024-09-18T01:17:49.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2328375296 bytes 2024-09-18T01:17:50.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2366126080 bytes 2024-09-18T01:17:50.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2463532032 bytes 2024-09-18T01:17:50.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2502593536 bytes 2024-09-18T01:17:50.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2532822016 bytes 2024-09-18T01:17:50.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2589415424 bytes 2024-09-18T01:17:50.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2669251584 bytes 2024-09-18T01:17:50.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2737324032 bytes 2024-09-18T01:17:51.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2814741504 bytes 2024-09-18T01:17:51.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2833312768 bytes 2024-09-18T01:17:51.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2915127296 bytes 2024-09-18T01:17:51.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2947810304 bytes 2024-09-18T01:17:51.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 3003979776 bytes 2024-09-18T01:17:51.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 3059729408 bytes 2024-09-18T01:17:51.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3060348928 bytes 2024-09-18T01:17:51.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3097936896 bytes 2024-09-18T01:17:52.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3148128256 bytes 2024-09-18T01:17:52.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3148712960 bytes 2024-09-18T01:17:52.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3192760320 bytes 2024-09-18T01:17:52.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3241690112 bytes 2024-09-18T01:17:52.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3242296320 bytes 2024-09-18T01:17:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3257596928 bytes 2024-09-18T01:17:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3282727936 bytes 2024-09-18T01:17:52.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3320198144 bytes 2024-09-18T01:17:52.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3339387904 bytes 2024-09-18T01:17:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3392855040 bytes 2024-09-18T01:17:53.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3445224448 bytes 2024-09-18T01:17:53.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3476893696 bytes 2024-09-18T01:17:53.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3525029888 bytes 2024-09-18T01:17:53.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3573021696 bytes 2024-09-18T01:17:53.686 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:17:53.685+0000 7f80d5d16640 -1 reset not still connected to 0x56418b8b4c30 2024-09-18T01:17:53.686 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:17:53.685+0000 7f80d5d16640 -1 reset not still connected to 0x5641963d4a90 2024-09-18T01:17:53.686 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:17:53.685+0000 7f80d5d16640 -1 reset not still connected to 0x56419c5909c0 2024-09-18T01:17:53.691 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:17:53.690+0000 7f80d5d16640 -1 reset not still connected to 0x5641a38e7450 2024-09-18T01:17:53.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3627707392 bytes 2024-09-18T01:17:53.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3696985088 bytes 2024-09-18T01:17:54.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3732736000 bytes 2024-09-18T01:17:54.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3764528128 bytes 2024-09-18T01:17:54.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3815278592 bytes 2024-09-18T01:17:54.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3871849472 bytes 2024-09-18T01:17:54.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3918665728 bytes 2024-09-18T01:17:54.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3964608512 bytes 2024-09-18T01:17:54.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 4002885632 bytes 2024-09-18T01:17:54.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4074118144 bytes 2024-09-18T01:17:55.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4090835968 bytes 2024-09-18T01:17:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4142544896 bytes 2024-09-18T01:17:55.363 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4194923520 bytes 2024-09-18T01:17:55.490 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4236551168 bytes 2024-09-18T01:17:55.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4314458112 bytes 2024-09-18T01:17:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4350139392 bytes 2024-09-18T01:17:55.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4411819008 bytes 2024-09-18T01:17:56.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4477329408 bytes 2024-09-18T01:17:56.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4515278848 bytes 2024-09-18T01:17:56.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4586869760 bytes 2024-09-18T01:17:56.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4639963136 bytes 2024-09-18T01:17:56.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4682863616 bytes 2024-09-18T01:17:56.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4720574464 bytes 2024-09-18T01:17:56.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4788209664 bytes 2024-09-18T01:17:56.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:17:56.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:17:56.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_47 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 snapshot 128M 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:17:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:56.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_47 128M 2024-09-18T01:17:56.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:56.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:56.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:56.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:56.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:17:56.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:56.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:17:56.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:17:56.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:17:56.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:17:56.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_47 2024-09-18T01:17:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:17:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:17:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:17:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_47 snapshot 2024-09-18T01:17:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:17:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:17:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_47 snapshot 2024-09-18T01:17:57.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:17:57.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_47 2024-09-18T01:17:58.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_47': 2024-09-18T01:17:58.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:17:58.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:17:58.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1437446cbb0 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1437446cbb0 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:17:57 2024 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:17:57 2024 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:17:57 2024 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:17:58.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: dc5f334a-10d4-4c3c-9345-0b305199e1f2 2024-09-18T01:17:58.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:17:58.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:17:58.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_47 2024-09-18T01:17:58.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:17:58.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:17:58.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:17:58.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:17:58.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=12 2024-09-18T01:17:58.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:17:58.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:17:58.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 101058560 bytes 2024-09-18T01:17:58.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 126069760 bytes 2024-09-18T01:17:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 145952768 bytes 2024-09-18T01:17:58.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 151597056 bytes 2024-09-18T01:17:58.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 200904704 bytes 2024-09-18T01:17:58.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 201513984 bytes 2024-09-18T01:17:58.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 264982528 bytes 2024-09-18T01:17:59.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 310277120 bytes 2024-09-18T01:17:59.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 365412352 bytes 2024-09-18T01:17:59.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 366022656 bytes 2024-09-18T01:17:59.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 372927488 bytes 2024-09-18T01:17:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 396745728 bytes 2024-09-18T01:17:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 479023104 bytes 2024-09-18T01:17:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 502437888 bytes 2024-09-18T01:17:59.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 524990464 bytes 2024-09-18T01:17:59.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 525592576 bytes 2024-09-18T01:18:00.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 603097088 bytes 2024-09-18T01:18:00.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 707227648 bytes 2024-09-18T01:18:00.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 841726976 bytes 2024-09-18T01:18:00.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 866630656 bytes 2024-09-18T01:18:00.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 887492608 bytes 2024-09-18T01:18:00.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 953877504 bytes 2024-09-18T01:18:01.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1004466176 bytes 2024-09-18T01:18:01.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1030437888 bytes 2024-09-18T01:18:01.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1057949696 bytes 2024-09-18T01:18:01.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1085767680 bytes 2024-09-18T01:18:01.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1142512640 bytes 2024-09-18T01:18:01.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1210984448 bytes 2024-09-18T01:18:01.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1231785984 bytes 2024-09-18T01:18:01.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1287660544 bytes 2024-09-18T01:18:02.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1338268672 bytes 2024-09-18T01:18:02.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1369443328 bytes 2024-09-18T01:18:02.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1412470784 bytes 2024-09-18T01:18:02.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1472546816 bytes 2024-09-18T01:18:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1543723008 bytes 2024-09-18T01:18:02.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1597925376 bytes 2024-09-18T01:18:02.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1629605888 bytes 2024-09-18T01:18:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1637502976 bytes 2024-09-18T01:18:03.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1688121344 bytes 2024-09-18T01:18:03.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1715181568 bytes 2024-09-18T01:18:03.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1804347392 bytes 2024-09-18T01:18:03.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1917532160 bytes 2024-09-18T01:18:03.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1923859456 bytes 2024-09-18T01:18:03.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1969197056 bytes 2024-09-18T01:18:04.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1977678848 bytes 2024-09-18T01:18:04.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2056195072 bytes 2024-09-18T01:18:04.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2075403264 bytes 2024-09-18T01:18:04.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2078511104 bytes 2024-09-18T01:18:04.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2141819904 bytes 2024-09-18T01:18:04.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2158135296 bytes 2024-09-18T01:18:04.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2161187840 bytes 2024-09-18T01:18:04.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2195058688 bytes 2024-09-18T01:18:05.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2285232128 bytes 2024-09-18T01:18:05.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2331913216 bytes 2024-09-18T01:18:05.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2371848192 bytes 2024-09-18T01:18:05.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2416214016 bytes 2024-09-18T01:18:05.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2426940416 bytes 2024-09-18T01:18:05.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2445362176 bytes 2024-09-18T01:18:05.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2525888512 bytes 2024-09-18T01:18:05.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2565883904 bytes 2024-09-18T01:18:06.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2616199168 bytes 2024-09-18T01:18:06.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2636726272 bytes 2024-09-18T01:18:06.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2639507456 bytes 2024-09-18T01:18:06.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2689293312 bytes 2024-09-18T01:18:06.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2788856832 bytes 2024-09-18T01:18:06.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2866344960 bytes 2024-09-18T01:18:06.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2901750784 bytes 2024-09-18T01:18:06.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2924376064 bytes 2024-09-18T01:18:07.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2975421440 bytes 2024-09-18T01:18:07.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3000958976 bytes 2024-09-18T01:18:07.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3041168384 bytes 2024-09-18T01:18:07.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3075619840 bytes 2024-09-18T01:18:07.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3086508032 bytes 2024-09-18T01:18:07.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3134050304 bytes 2024-09-18T01:18:07.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3182491648 bytes 2024-09-18T01:18:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3214491648 bytes 2024-09-18T01:18:08.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3271702528 bytes 2024-09-18T01:18:08.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3316599808 bytes 2024-09-18T01:18:08.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3324698624 bytes 2024-09-18T01:18:08.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3408717824 bytes 2024-09-18T01:18:08.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3451858944 bytes 2024-09-18T01:18:08.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3479989248 bytes 2024-09-18T01:18:08.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3529489408 bytes 2024-09-18T01:18:08.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3584594944 bytes 2024-09-18T01:18:09.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3592371200 bytes 2024-09-18T01:18:09.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3609568256 bytes 2024-09-18T01:18:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3662240768 bytes 2024-09-18T01:18:09.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3686994944 bytes 2024-09-18T01:18:09.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3705610240 bytes 2024-09-18T01:18:09.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3741612032 bytes 2024-09-18T01:18:09.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3793318912 bytes 2024-09-18T01:18:09.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3838922752 bytes 2024-09-18T01:18:10.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:18:10.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:18:10.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:18:10.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_48 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 snapshot 128M 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:10.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_48 128M 2024-09-18T01:18:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:18:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:18:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:18:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:18:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:10.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_48 2024-09-18T01:18:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:18:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:18:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:18:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_48 snapshot 2024-09-18T01:18:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:18:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:18:10.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_48 snapshot 2024-09-18T01:18:10.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:18:10.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_48 2024-09-18T01:18:11.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_48': 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 14434ce4995d 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.14434ce4995d 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:18:10 2024 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:18:10 2024 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:18:10 2024 2024-09-18T01:18:11.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:18:11.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:18:11.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: db673391-3699-4385-93be-31f993e37bad 2024-09-18T01:18:11.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:18:11.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:18:11.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_48 2024-09-18T01:18:11.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:11.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:11.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:18:11.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:18:11.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=12 2024-09-18T01:18:11.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:18:11.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:18:11.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 39797760 bytes 2024-09-18T01:18:11.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 80833536 bytes 2024-09-18T01:18:11.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 173160448 bytes 2024-09-18T01:18:11.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 221902848 bytes 2024-09-18T01:18:11.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 226870272 bytes 2024-09-18T01:18:11.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 297736192 bytes 2024-09-18T01:18:12.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 416935936 bytes 2024-09-18T01:18:12.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 481220608 bytes 2024-09-18T01:18:12.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 526695424 bytes 2024-09-18T01:18:12.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 555463680 bytes 2024-09-18T01:18:12.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 600285184 bytes 2024-09-18T01:18:12.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 639047680 bytes 2024-09-18T01:18:12.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 683606016 bytes 2024-09-18T01:18:13.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 761954304 bytes 2024-09-18T01:18:13.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 788063232 bytes 2024-09-18T01:18:13.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 875380736 bytes 2024-09-18T01:18:13.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 980586496 bytes 2024-09-18T01:18:13.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 1010752512 bytes 2024-09-18T01:18:13.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 1022113792 bytes 2024-09-18T01:18:13.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 1034312704 bytes 2024-09-18T01:18:13.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1091018752 bytes 2024-09-18T01:18:14.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1128671232 bytes 2024-09-18T01:18:14.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1140147200 bytes 2024-09-18T01:18:14.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1172908032 bytes 2024-09-18T01:18:14.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1200801792 bytes 2024-09-18T01:18:14.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1236345856 bytes 2024-09-18T01:18:14.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1265505280 bytes 2024-09-18T01:18:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1276136448 bytes 2024-09-18T01:18:14.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1297052672 bytes 2024-09-18T01:18:15.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1385079808 bytes 2024-09-18T01:18:15.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1457293312 bytes 2024-09-18T01:18:15.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1486331904 bytes 2024-09-18T01:18:15.458 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1537245184 bytes 2024-09-18T01:18:15.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1569572864 bytes 2024-09-18T01:18:15.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1570206720 bytes 2024-09-18T01:18:15.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1638734848 bytes 2024-09-18T01:18:15.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1683976192 bytes 2024-09-18T01:18:16.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1705583616 bytes 2024-09-18T01:18:16.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1753266176 bytes 2024-09-18T01:18:16.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1768912896 bytes 2024-09-18T01:18:16.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1825764352 bytes 2024-09-18T01:18:16.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1850273792 bytes 2024-09-18T01:18:16.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1855610880 bytes 2024-09-18T01:18:16.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1961115648 bytes 2024-09-18T01:18:16.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2022794240 bytes 2024-09-18T01:18:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2036017152 bytes 2024-09-18T01:18:17.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2075380736 bytes 2024-09-18T01:18:17.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2105654272 bytes 2024-09-18T01:18:17.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2172404736 bytes 2024-09-18T01:18:17.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2213827584 bytes 2024-09-18T01:18:17.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2224560128 bytes 2024-09-18T01:18:17.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2291792896 bytes 2024-09-18T01:18:17.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2328401920 bytes 2024-09-18T01:18:18.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2354783232 bytes 2024-09-18T01:18:18.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2376078336 bytes 2024-09-18T01:18:18.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2387955712 bytes 2024-09-18T01:18:18.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2431803392 bytes 2024-09-18T01:18:18.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2487405568 bytes 2024-09-18T01:18:18.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2550298624 bytes 2024-09-18T01:18:18.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2599037952 bytes 2024-09-18T01:18:18.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2610015232 bytes 2024-09-18T01:18:19.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2631973888 bytes 2024-09-18T01:18:19.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2689840128 bytes 2024-09-18T01:18:19.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2719593472 bytes 2024-09-18T01:18:19.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2780496896 bytes 2024-09-18T01:18:19.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2878672896 bytes 2024-09-18T01:18:19.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2913101824 bytes 2024-09-18T01:18:19.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2960266240 bytes 2024-09-18T01:18:20.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3004525568 bytes 2024-09-18T01:18:20.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3035629568 bytes 2024-09-18T01:18:20.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3109662720 bytes 2024-09-18T01:18:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3134002176 bytes 2024-09-18T01:18:20.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3202535424 bytes 2024-09-18T01:18:20.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3241794560 bytes 2024-09-18T01:18:20.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3275140096 bytes 2024-09-18T01:18:20.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3287231488 bytes 2024-09-18T01:18:21.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3294637056 bytes 2024-09-18T01:18:21.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3372590080 bytes 2024-09-18T01:18:21.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3406213120 bytes 2024-09-18T01:18:21.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3442917376 bytes 2024-09-18T01:18:21.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3446378496 bytes 2024-09-18T01:18:21.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3508368384 bytes 2024-09-18T01:18:21.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3555326976 bytes 2024-09-18T01:18:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3627034624 bytes 2024-09-18T01:18:22.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3688520704 bytes 2024-09-18T01:18:22.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3718437888 bytes 2024-09-18T01:18:22.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3743023104 bytes 2024-09-18T01:18:22.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3760441344 bytes 2024-09-18T01:18:22.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3799180288 bytes 2024-09-18T01:18:22.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3876005888 bytes 2024-09-18T01:18:22.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3902673920 bytes 2024-09-18T01:18:23.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:18:23.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:18:23.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:18:23.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:18:23.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:18:23.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_49 2024-09-18T01:18:23.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 snapshot 128M 2024-09-18T01:18:23.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:23.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:23.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:18:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:18:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:18:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:23.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_49 128M 2024-09-18T01:18:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:18:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:18:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:18:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:18:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:23.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_49 2024-09-18T01:18:23.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:18:23.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:23.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:18:23.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:18:23.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_49 snapshot 2024-09-18T01:18:23.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:23.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:23.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:18:23.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:18:23.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_49 snapshot 2024-09-18T01:18:23.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:18:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_49 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_49': 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 144f63a65e79 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.144f63a65e79 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:18:23 2024 2024-09-18T01:18:23.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:18:23 2024 2024-09-18T01:18:23.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:18:23 2024 2024-09-18T01:18:23.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:18:23.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:18:23.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 380bbc6e-0c06-47e3-a103-ff2ba6ff04d9 2024-09-18T01:18:23.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:18:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:18:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_49 2024-09-18T01:18:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:18:23.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:18:23.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=11 2024-09-18T01:18:23.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:18:23.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:18:23.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 89945088 bytes 2024-09-18T01:18:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 160029696 bytes 2024-09-18T01:18:23.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 187389952 bytes 2024-09-18T01:18:23.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 234059776 bytes 2024-09-18T01:18:24.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 277874688 bytes 2024-09-18T01:18:24.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 295040000 bytes 2024-09-18T01:18:24.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 391942144 bytes 2024-09-18T01:18:24.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 418969600 bytes 2024-09-18T01:18:24.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 444432384 bytes 2024-09-18T01:18:24.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 450857984 bytes 2024-09-18T01:18:24.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 503452672 bytes 2024-09-18T01:18:24.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 515814400 bytes 2024-09-18T01:18:24.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 581804032 bytes 2024-09-18T01:18:25.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 612152320 bytes 2024-09-18T01:18:25.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 646019072 bytes 2024-09-18T01:18:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 735433728 bytes 2024-09-18T01:18:25.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 736049152 bytes 2024-09-18T01:18:25.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 808189952 bytes 2024-09-18T01:18:25.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 878128128 bytes 2024-09-18T01:18:25.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 902073344 bytes 2024-09-18T01:18:26.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 984121344 bytes 2024-09-18T01:18:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1037237248 bytes 2024-09-18T01:18:26.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1080278016 bytes 2024-09-18T01:18:26.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1119773696 bytes 2024-09-18T01:18:26.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1120401408 bytes 2024-09-18T01:18:26.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1160073216 bytes 2024-09-18T01:18:26.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1216510976 bytes 2024-09-18T01:18:26.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1237368832 bytes 2024-09-18T01:18:27.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1257888768 bytes 2024-09-18T01:18:27.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1314791424 bytes 2024-09-18T01:18:27.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1340271616 bytes 2024-09-18T01:18:27.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1353374720 bytes 2024-09-18T01:18:27.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1385391104 bytes 2024-09-18T01:18:27.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1427799040 bytes 2024-09-18T01:18:27.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1486621696 bytes 2024-09-18T01:18:27.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1505626112 bytes 2024-09-18T01:18:28.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1572617216 bytes 2024-09-18T01:18:28.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1641592832 bytes 2024-09-18T01:18:28.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1661765632 bytes 2024-09-18T01:18:28.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1690953728 bytes 2024-09-18T01:18:28.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1749285888 bytes 2024-09-18T01:18:28.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1826280448 bytes 2024-09-18T01:18:28.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1832484864 bytes 2024-09-18T01:18:29.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1870325760 bytes 2024-09-18T01:18:29.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1902654464 bytes 2024-09-18T01:18:29.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1955375104 bytes 2024-09-18T01:18:29.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1979392000 bytes 2024-09-18T01:18:29.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2030593024 bytes 2024-09-18T01:18:29.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2046917632 bytes 2024-09-18T01:18:29.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2090613760 bytes 2024-09-18T01:18:29.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2134759424 bytes 2024-09-18T01:18:30.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2209541120 bytes 2024-09-18T01:18:30.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2248064000 bytes 2024-09-18T01:18:30.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2289001472 bytes 2024-09-18T01:18:30.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2368650240 bytes 2024-09-18T01:18:30.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2438759424 bytes 2024-09-18T01:18:30.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2439407616 bytes 2024-09-18T01:18:30.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2463627264 bytes 2024-09-18T01:18:31.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2515511296 bytes 2024-09-18T01:18:31.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2531632128 bytes 2024-09-18T01:18:31.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2565962752 bytes 2024-09-18T01:18:31.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2595733504 bytes 2024-09-18T01:18:31.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2645471232 bytes 2024-09-18T01:18:31.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2652198912 bytes 2024-09-18T01:18:31.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2696541184 bytes 2024-09-18T01:18:31.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2697145344 bytes 2024-09-18T01:18:31.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2778916864 bytes 2024-09-18T01:18:32.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2797416448 bytes 2024-09-18T01:18:32.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2849352704 bytes 2024-09-18T01:18:32.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2903489536 bytes 2024-09-18T01:18:32.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2996609024 bytes 2024-09-18T01:18:32.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3031600128 bytes 2024-09-18T01:18:32.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3047787520 bytes 2024-09-18T01:18:32.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3080070144 bytes 2024-09-18T01:18:33.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3116710912 bytes 2024-09-18T01:18:33.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3144168448 bytes 2024-09-18T01:18:33.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3179005952 bytes 2024-09-18T01:18:33.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3203743744 bytes 2024-09-18T01:18:33.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3289049088 bytes 2024-09-18T01:18:33.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3353044992 bytes 2024-09-18T01:18:33.815 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3381122048 bytes 2024-09-18T01:18:33.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3445213184 bytes 2024-09-18T01:18:34.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3457364992 bytes 2024-09-18T01:18:34.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3534464000 bytes 2024-09-18T01:18:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:18:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:18:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:18:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:18:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:18:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_50 2024-09-18T01:18:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 snapshot 128M 2024-09-18T01:18:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:34.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:34.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:18:34.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:34.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:18:34.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n snapshot ']' 2024-09-18T01:18:34.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:34.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_50 128M 2024-09-18T01:18:34.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:34.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:34.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:34.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:34.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:18:34.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:34.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:18:34.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 128M ']' 2024-09-18T01:18:34.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=128M 2024-09-18T01:18:34.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:18:34.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock mirror/image_50 2024-09-18T01:18:34.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ is_pool_mirror_mode_image mirror 2024-09-18T01:18:34.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' image = image ']' 2024-09-18T01:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_50 snapshot 2024-09-18T01:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:18:34.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_50 snapshot 2024-09-18T01:18:35.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:18:35.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_50 2024-09-18T01:18:35.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_50': 2024-09-18T01:18:35.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:18:35.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:18:35.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:18:35.511 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 145b1ef77cc5 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.145b1ef77cc5 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:18:34 2024 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:18:34 2024 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:18:34 2024 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: e589d353-55de-4143-8072-8a9ace421dbf 2024-09-18T01:18:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:18:35.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n '' ']' 2024-09-18T01:18:35.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_50 2024-09-18T01:18:35.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:35.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:35.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:18:35.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:18:35.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:18:35.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:18:35.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:18:35.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 21085184 bytes 2024-09-18T01:18:35.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 45762560 bytes 2024-09-18T01:18:35.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 59381760 bytes 2024-09-18T01:18:36.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 143942656 bytes 2024-09-18T01:18:36.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 210187264 bytes 2024-09-18T01:18:36.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 270662656 bytes 2024-09-18T01:18:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 314201088 bytes 2024-09-18T01:18:36.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 346185728 bytes 2024-09-18T01:18:36.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 385053696 bytes 2024-09-18T01:18:36.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 390077440 bytes 2024-09-18T01:18:36.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 440754176 bytes 2024-09-18T01:18:37.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 504975360 bytes 2024-09-18T01:18:37.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 527087616 bytes 2024-09-18T01:18:37.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 559241216 bytes 2024-09-18T01:18:37.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 586273792 bytes 2024-09-18T01:18:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 679305216 bytes 2024-09-18T01:18:37.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 708915200 bytes 2024-09-18T01:18:37.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 787861504 bytes 2024-09-18T01:18:38.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 837953536 bytes 2024-09-18T01:18:38.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 886961152 bytes 2024-09-18T01:18:38.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 935902208 bytes 2024-09-18T01:18:38.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 936523776 bytes 2024-09-18T01:18:38.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1019525120 bytes 2024-09-18T01:18:38.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1096795136 bytes 2024-09-18T01:18:38.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1137558528 bytes 2024-09-18T01:18:39.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1165850624 bytes 2024-09-18T01:18:39.151 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1254425600 bytes 2024-09-18T01:18:39.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1282610176 bytes 2024-09-18T01:18:39.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1329814528 bytes 2024-09-18T01:18:39.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1374462976 bytes 2024-09-18T01:18:39.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1448788992 bytes 2024-09-18T01:18:39.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1466973184 bytes 2024-09-18T01:18:39.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1528902656 bytes 2024-09-18T01:18:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1555117056 bytes 2024-09-18T01:18:40.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1600363520 bytes 2024-09-18T01:18:40.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1690138624 bytes 2024-09-18T01:18:40.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1732902912 bytes 2024-09-18T01:18:40.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1733522432 bytes 2024-09-18T01:18:40.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1758274560 bytes 2024-09-18T01:18:40.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1849754624 bytes 2024-09-18T01:18:41.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1879520256 bytes 2024-09-18T01:18:41.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1883718656 bytes 2024-09-18T01:18:41.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1968470016 bytes 2024-09-18T01:18:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2127502336 bytes 2024-09-18T01:18:41.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2188686336 bytes 2024-09-18T01:18:41.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2219086848 bytes 2024-09-18T01:18:41.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2262562816 bytes 2024-09-18T01:18:42.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2313905152 bytes 2024-09-18T01:18:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2314510336 bytes 2024-09-18T01:18:42.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2349392896 bytes 2024-09-18T01:18:42.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2402433024 bytes 2024-09-18T01:18:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2488368128 bytes 2024-09-18T01:18:42.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2515929088 bytes 2024-09-18T01:18:42.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2630457344 bytes 2024-09-18T01:18:43.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2679207936 bytes 2024-09-18T01:18:43.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2749326336 bytes 2024-09-18T01:18:43.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2767632384 bytes 2024-09-18T01:18:43.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2805228544 bytes 2024-09-18T01:18:43.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2870868992 bytes 2024-09-18T01:18:43.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2978305024 bytes 2024-09-18T01:18:43.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 3011419136 bytes 2024-09-18T01:18:44.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 3067610112 bytes 2024-09-18T01:18:44.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 3180045312 bytes 2024-09-18T01:18:44.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 3222533120 bytes 2024-09-18T01:18:44.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 3257634816 bytes 2024-09-18T01:18:44.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 3267734528 bytes 2024-09-18T01:18:44.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3354036224 bytes 2024-09-18T01:18:44.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3401935872 bytes 2024-09-18T01:18:45.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3475274752 bytes 2024-09-18T01:18:45.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3491777536 bytes 2024-09-18T01:18:45.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3516587008 bytes 2024-09-18T01:18:45.443 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3564573696 bytes 2024-09-18T01:18:45.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3636674560 bytes 2024-09-18T01:18:45.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3751103488 bytes 2024-09-18T01:18:45.938 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3807421440 bytes 2024-09-18T01:18:46.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3887139840 bytes 2024-09-18T01:18:46.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3917876224 bytes 2024-09-18T01:18:46.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3972923392 bytes 2024-09-18T01:18:46.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 4046841856 bytes 2024-09-18T01:18:46.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 4116819968 bytes 2024-09-18T01:18:46.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 4155606016 bytes 2024-09-18T01:18:46.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 4190693376 bytes 2024-09-18T01:18:47.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 4253822976 bytes 2024-09-18T01:18:47.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 4271609856 bytes 2024-09-18T01:18:47.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 4290658304 bytes 2024-09-18T01:18:47.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 4315368448 bytes 2024-09-18T01:18:47.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 4344522752 bytes 2024-09-18T01:18:47.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 4459905024 bytes 2024-09-18T01:18:47.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 4495377408 bytes 2024-09-18T01:18:48.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 4561104896 bytes 2024-09-18T01:18:48.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 4572421120 bytes 2024-09-18T01:18:48.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4577193984 bytes 2024-09-18T01:18:48.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4675405824 bytes 2024-09-18T01:18:48.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4735026176 bytes 2024-09-18T01:18:48.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4736631808 bytes 2024-09-18T01:18:48.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4854773760 bytes 2024-09-18T01:18:48.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4922602496 bytes 2024-09-18T01:18:49.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4997339136 bytes 2024-09-18T01:18:49.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 5012501504 bytes 2024-09-18T01:18:49.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 5065033728 bytes 2024-09-18T01:18:49.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_pool_images cluster2 mirror 50 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_count=50 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local count 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_count=0 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:18:49.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 40 2024-09-18T01:18:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in `seq 1 40` 2024-09-18T01:18:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 1 -ne 1 2024-09-18T01:18:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2024-09-18T01:18:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep 'images: ' 2024-09-18T01:18:49.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cut '-d ' -f 2 2024-09-18T01:18:49.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: failed to query services: (13) Permission denied 2024-09-18T01:18:49.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ count=50 2024-09-18T01:18:49.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 50 = 50 2024-09-18T01:18:49.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:18:49.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_pool_healthy cluster2 mirror 2024-09-18T01:18:49.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:18:49.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:49.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:18:49.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state 2024-09-18T01:18:49.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 40 2024-09-18T01:18:49.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in `seq 1 40` 2024-09-18T01:18:49.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 1 -ne 1 2024-09-18T01:18:49.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2024-09-18T01:18:49.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep 'image health:' 2024-09-18T01:18:49.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cut '-d ' -f 3 2024-09-18T01:18:49.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: failed to query services: (13) Permission denied 2024-09-18T01:18:49.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=OK 2024-09-18T01:18:49.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test OK = ERROR 2024-09-18T01:18:49.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test OK = OK 2024-09-18T01:18:49.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:18:49.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_pool_images cluster1 mirror 50 2024-09-18T01:18:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:18:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_count=50 2024-09-18T01:18:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:18:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local count 2024-09-18T01:18:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_count=0 2024-09-18T01:18:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:18:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 40 2024-09-18T01:18:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in `seq 1 40` 2024-09-18T01:18:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 1 -ne 1 2024-09-18T01:18:49.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-18T01:18:49.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep 'images: ' 2024-09-18T01:18:49.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cut '-d ' -f 2 2024-09-18T01:18:49.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: failed to query services: (13) Permission denied 2024-09-18T01:18:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ count=50 2024-09-18T01:18:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 50 = 50 2024-09-18T01:18:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:18:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_pool_healthy cluster1 mirror 2024-09-18T01:18:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:18:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:18:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:18:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state 2024-09-18T01:18:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 40 2024-09-18T01:18:49.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in `seq 1 40` 2024-09-18T01:18:49.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 1 -ne 1 2024-09-18T01:18:49.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-18T01:18:49.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep 'image health:' 2024-09-18T01:18:49.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cut '-d ' -f 3 2024-09-18T01:18:49.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: failed to query services: (13) Permission denied 2024-09-18T01:18:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=WARNING 2024-09-18T01:18:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test WARNING = ERROR 2024-09-18T01:18:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test WARNING = OK 2024-09-18T01:18:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in `seq 1 40` 2024-09-18T01:18:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 2 -ne 1 2024-09-18T01:18:49.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 30 2024-09-18T01:19:08.447 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.44959069395250495) > weight (0.25) 2024-09-18T01:19:08.447 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-18T01:19:08.448 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-18T01:19:08.448 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-09-18T01:19:08.449 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:19:08.447+0000 7f76a3dbe640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 37005) UID: 1000 2024-09-18T01:19:09.590 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.589+0000 7f636fc8d640 -1 rbd::mirror::ImageReplayer: 0x563146f2aa00 [3/820484e1-ad03-4296-9676-f12697dabdb5] start: already running 2024-09-18T01:19:09.598 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f637048e640 -1 rbd::mirror::ImageReplayer: 0x5631471c8280 [3/7af955aa-6145-46fc-be77-ed5d53ea99dd] start: already running 2024-09-18T01:19:09.598 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f637048e640 -1 rbd::mirror::ImageReplayer: 0x5631471c8000 [3/6f579a7e-81b1-4594-9e4f-741a03c86452] start: already running 2024-09-18T01:19:09.598 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f637048e640 -1 rbd::mirror::ImageReplayer: 0x563146f2b900 [3/bf14a2b6-1625-4fba-8a13-3e69c7467ad0] start: already running 2024-09-18T01:19:09.599 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f3640e8e640 -1 rbd::mirror::ImageReplayer: 0x557ef579cf00 [3/e589d353-55de-4143-8072-8a9ace421dbf] start: already running 2024-09-18T01:19:09.599 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f364d6a7640 -1 rbd::mirror::ImageReplayer: 0x557ef579cc80 [3/aeaeb058-01f1-4690-999c-8c21dad1561e] start: already running 2024-09-18T01:19:09.599 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f3640e8e640 -1 rbd::mirror::ImageReplayer: 0x557ef4b4af00 [3/6c6c64a2-50e5-44f2-8703-2da1463fc540] start: already running 2024-09-18T01:19:09.599 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f3640e8e640 -1 rbd::mirror::ImageReplayer: 0x557ef4b4aa00 [3/75569094-ed91-4db4-95b0-2512a6bdcc51] start: already running 2024-09-18T01:19:09.599 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f3640e8e640 -1 rbd::mirror::ImageReplayer: 0x557ef579d400 [3/f06096fd-c27d-4c66-a888-be876f867f5a] start: already running 2024-09-18T01:19:09.599 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f3640e8e640 -1 rbd::mirror::ImageReplayer: 0x557ef4b4b680 [3/bf72281c-0895-415a-a775-377773847714] start: already running 2024-09-18T01:19:09.599 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f3640e8e640 -1 rbd::mirror::ImageReplayer: 0x557ef4b4b400 [3/957171f2-b93c-457d-89be-5d45fdaad96c] start: already running 2024-09-18T01:19:09.599 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f364d6a7640 -1 rbd::mirror::ImageReplayer: 0x557ef4b4a780 [3/884d933d-d88c-4657-97ca-5aa2e3991921] start: already running 2024-09-18T01:19:09.599 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:19:09.596+0000 7f3640e8e640 -1 rbd::mirror::ImageReplayer: 0x557ef4b4b900 [3/d6c020f1-f8ac-4074-858c-a7dfb0ecf52e] start: already running 2024-09-18T01:19:09.601 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.599+0000 7f637d4a8640 -1 rbd::mirror::ImageReplayer: 0x563146f2ac80 [3/dc5f334a-10d4-4c3c-9345-0b305199e1f2] start: already running 2024-09-18T01:19:09.601 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.599+0000 7f637d4a8640 -1 rbd::mirror::ImageReplayer: 0x563146f2b400 [3/716a32d0-d4de-4f8b-8c5b-1431e7a8c859] start: already running 2024-09-18T01:19:09.601 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.599+0000 7f637d4a8640 -1 rbd::mirror::ImageReplayer: 0x563146f2b180 [3/e2f372b0-91b5-4e89-95d5-57ab346b50a4] start: already running 2024-09-18T01:19:09.601 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.599+0000 7f637d4a8640 -1 rbd::mirror::ImageReplayer: 0x563146f2a280 [3/0efe7b51-8742-400d-adca-bf1d51dea26b] start: already running 2024-09-18T01:19:09.602 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.599+0000 7f637d4a8640 -1 rbd::mirror::ImageReplayer: 0x563146f2b680 [3/753ffe95-b234-4f46-a463-e642f6aa8751] start: already running 2024-09-18T01:19:09.602 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.599+0000 7f637d4a8640 -1 rbd::mirror::ImageReplayer: 0x563146f2bb80 [3/855097c3-5440-41cd-a767-29e218b7cf40] start: already running 2024-09-18T01:19:09.602 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.599+0000 7f637d4a8640 -1 rbd::mirror::ImageReplayer: 0x563146f2a780 [3/ca67ff60-0889-4b07-bb52-b0e5f8ac56c7] start: already running 2024-09-18T01:19:09.602 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:19:09.599+0000 7f637d4a8640 -1 rbd::mirror::ImageReplayer: 0x563146f2af00 [3/26ca4ff3-0678-4f67-8c75-7af181831497] start: already running 2024-09-18T01:19:16.449 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-18T01:19:16.449 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:19:16.449 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-18T01:19:16.450 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-18T01:19:16.450 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-18T01:19:16.453 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-18T01:19:16.453 INFO:tasks.rbd_mirror_thrash:waiting for 71 secs before thrashing 2024-09-18T01:19:19.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-18T01:19:19.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep 'image health:' 2024-09-18T01:19:19.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cut '-d ' -f 3 2024-09-18T01:19:19.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: failed to query services: (13) Permission denied 2024-09-18T01:19:19.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=OK 2024-09-18T01:19:19.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test OK = ERROR 2024-09-18T01:19:19.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test OK = OK 2024-09-18T01:19:19.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:19:19.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ testlog 'TEST: compare many images' 2024-09-18T01:19:19.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ tee -a /tmp/tmp.yeNdCEAq6i/rbd-mirror.test.log 2024-09-18T01:19:19.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ date '+%F %T' 2024-09-18T01:19:19.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo 2024-09-18 01:19:19 TEST: compare many images 2024-09-18T01:19:19.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:2024-09-18 01:19:19 TEST: compare many images 2024-09-18T01:19:19.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 50 2024-09-18T01:19:19.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:19:19.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_1 2024-09-18T01:19:19.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_1 snap 2024-09-18T01:19:19.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:19:19.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:19.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:19.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:19:19.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:19:21.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_1 2024-09-18T01:19:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started 2024-09-18T01:19:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:21.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:19:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:19:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:19:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:19:22.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_1 started 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:19:22.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:19:22.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:19:22.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:19:22.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:19:22.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:19:22.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:19:22.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:22.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2024-09-18T01:19:22.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:19:22.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:22.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:22.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:19:22.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:19:22.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:22.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:19:22.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:19:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:23.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2024-09-18T01:19:23.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 5 closest matches: 2024-09-18T01:19:23.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_11 2024-09-18T01:19:23.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_12 2024-09-18T01:19:23.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_13 2024-09-18T01:19:23.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_15 2024-09-18T01:19:23.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_16 2024-09-18T01:19:23.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2024-09-18T01:19:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:19:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:19:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:19:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:23.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:23.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:23.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:23.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:23.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:23.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:23.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:19:23.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:19:23.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:19:23.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:23.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:23.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:23.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2024-09-18T01:19:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 1 closest matches: 2024-09-18T01:19:23.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_19 2024-09-18T01:19:23.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_1 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:23.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:19:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:19:23.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:19:23.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:23.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:23.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:23.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_1 2024-09-18T01:19:23.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 1 closest matches: 2024-09-18T01:19:23.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_18 2024-09-18T01:19:23.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_1 2024-09-18T01:19:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:19:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:23.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:23.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:19:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:19:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:23.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_1 2024-09-18T01:19:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 2 closest matches: 2024-09-18T01:19:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_10 2024-09-18T01:19:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_14 2024-09-18T01:19:23.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:23.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:19:23.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 1 2024-09-18T01:19:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:19:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:19:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:24.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2024-09-18T01:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:19:24.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:24.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:19:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:19:24.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:19:24.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:24.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:24.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:24.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2024-09-18T01:19:24.515 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.476450138018244) > weight (0.25) 2024-09-18T01:19:24.515 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.338542715347005) > weight (0.25) 2024-09-18T01:19:24.515 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.9698068580565486) > weight (0.25) 2024-09-18T01:19:24.515 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.4249969656268785) > weight (0.25) 2024-09-18T01:19:24.515 INFO:tasks.rbd_mirror_thrash:waiting for 75 secs before thrashing 2024-09-18T01:19:24.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 5 closest matches: 2024-09-18T01:19:24.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_11 2024-09-18T01:19:24.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_12 2024-09-18T01:19:24.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_13 2024-09-18T01:19:24.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_15 2024-09-18T01:19:24.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_16 2024-09-18T01:19:24.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:24.548 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:24.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2024-09-18T01:19:24.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:19:24.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:24.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:24.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:19:24.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:19:24.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:24.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:24.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:24.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:24.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:24.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:24.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:24.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:24.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:24.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:24.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:19:24.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:19:24.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:19:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:24.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2024-09-18T01:19:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 1 closest matches: 2024-09-18T01:19:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_19 2024-09-18T01:19:24.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:24.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:24.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_1 2024-09-18T01:19:24.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:19:24.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:24.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:19:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:19:24.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:19:24.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:24.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:24.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:24.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_1 2024-09-18T01:19:24.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 1 closest matches: 2024-09-18T01:19:24.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_18 2024-09-18T01:19:24.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:24.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:24.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_1 2024-09-18T01:19:24.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:19:24.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:24.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:24.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:24.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:19:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:19:24.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:24.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:24.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:24.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_1 2024-09-18T01:19:24.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 2 closest matches: 2024-09-18T01:19:24.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_10 2024-09-18T01:19:24.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_14 2024-09-18T01:19:24.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:24.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:19:24.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 2 2024-09-18T01:19:26.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:19:26.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:19:26.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:26.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2024-09-18T01:19:26.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:19:26.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:26.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:26.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:19:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:19:26.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:19:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:26.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2024-09-18T01:19:26.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 5 closest matches: 2024-09-18T01:19:26.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_11 2024-09-18T01:19:26.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_12 2024-09-18T01:19:26.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_13 2024-09-18T01:19:26.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_15 2024-09-18T01:19:26.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_16 2024-09-18T01:19:26.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2024-09-18T01:19:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:19:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:26.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:19:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:19:26.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:19:26.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:26.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:26.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:26.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2024-09-18T01:19:27.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 1 closest matches: 2024-09-18T01:19:27.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_19 2024-09-18T01:19:27.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:27.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:27.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_1 2024-09-18T01:19:27.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:19:27.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:27.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:27.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:19:27.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:19:27.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:27.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:19:27.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:19:27.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:27.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:27.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:27.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:27.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:27.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:27.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:27.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:19:27.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:19:27.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:19:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_1 2024-09-18T01:19:27.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 1 closest matches: 2024-09-18T01:19:27.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_18 2024-09-18T01:19:27.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:27.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:27.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_1 2024-09-18T01:19:27.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:19:27.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:19:27.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:19:27.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:19:27.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:19:27.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:27.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:27.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:27.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_1 2024-09-18T01:19:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 2 closest matches: 2024-09-18T01:19:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_10 2024-09-18T01:19:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_14 2024-09-18T01:19:27.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:27.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:19:27.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 4 2024-09-18T01:19:31.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:19:31.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:19:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2024-09-18T01:19:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:19:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:19:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:19:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:31.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:19:31.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:31.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:31.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:31.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:31.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:31.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:31.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:31.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:19:31.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:19:31.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:19:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:31.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:31.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2024-09-18T01:19:31.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 5 closest matches: 2024-09-18T01:19:31.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_11 2024-09-18T01:19:31.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_12 2024-09-18T01:19:31.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_13 2024-09-18T01:19:31.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_15 2024-09-18T01:19:31.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_16 2024-09-18T01:19:31.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2024-09-18T01:19:31.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:19:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:19:31.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:19:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:31.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:19:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:19:31.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:31.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:31.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:31.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2024-09-18T01:19:31.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 1 closest matches: 2024-09-18T01:19:31.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_19 2024-09-18T01:19:31.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_1 2024-09-18T01:19:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:19:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:19:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:19:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:31.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:19:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:19:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:19:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:19:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:31.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:19:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:19:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:19:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:19:31.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:19:31.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:31.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:31.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:31.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_1 2024-09-18T01:19:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 1 closest matches: 2024-09-18T01:19:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_18 2024-09-18T01:19:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:31.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:31.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_1 2024-09-18T01:19:31.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:19:31.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:19:31.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:31.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:31.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:19:31.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:19:31.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:19:31.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:31.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:31.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:31.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_1 2024-09-18T01:19:31.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 2 closest matches: 2024-09-18T01:19:31.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_10 2024-09-18T01:19:31.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_14 2024-09-18T01:19:31.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:31.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:19:31.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 8 2024-09-18T01:19:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:19:39.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:19:39.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:39.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2024-09-18T01:19:39.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:19:39.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:39.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:39.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:19:39.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:39.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:19:39.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:19:39.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:39.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:39.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:39.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:39.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:39.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:39.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:39.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:19:39.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:19:39.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:19:39.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:39.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:39.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:39.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2024-09-18T01:19:39.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 5 closest matches: 2024-09-18T01:19:39.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_11 2024-09-18T01:19:39.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_12 2024-09-18T01:19:39.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_13 2024-09-18T01:19:39.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_15 2024-09-18T01:19:39.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_16 2024-09-18T01:19:39.815 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2024-09-18T01:19:39.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:39.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:19:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:19:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:19:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:19:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:39.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:39.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:39.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:19:39.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:19:39.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:19:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:39.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:39.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2024-09-18T01:19:39.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 1 closest matches: 2024-09-18T01:19:39.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_19 2024-09-18T01:19:39.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_1 2024-09-18T01:19:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:19:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:19:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:19:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:39.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:19:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:19:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:19:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:19:39.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:39.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:39.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:39.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_1 2024-09-18T01:19:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 1 closest matches: 2024-09-18T01:19:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_18 2024-09-18T01:19:40.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_1 2024-09-18T01:19:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:19:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:19:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:19:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:19:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:19:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:19:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:19:40.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:19:40.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:40.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:19:40.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_1 2024-09-18T01:19:40.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 2 closest matches: 2024-09-18T01:19:40.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_10 2024-09-18T01:19:40.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_14 2024-09-18T01:19:40.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:40.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:19:40.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 8 2024-09-18T01:19:48.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:19:48.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:19:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2024-09-18T01:19:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:19:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:19:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:19:48.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:19:48.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:19:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:19:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:19:48.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:19:48.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:48.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:48.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:48.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2024-09-18T01:19:48.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 5 closest matches: 2024-09-18T01:19:48.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_11 2024-09-18T01:19:48.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_12 2024-09-18T01:19:48.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_13 2024-09-18T01:19:48.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_15 2024-09-18T01:19:48.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_16 2024-09-18T01:19:48.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:48.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:48.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:48.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:48.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:48.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:48.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:19:48.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:19:48.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:19:48.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:48.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:48.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:48.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2024-09-18T01:19:48.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:19:48.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:19:48.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:19:48.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:19:48.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:19:48.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:19:48.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:19:48.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:19:48.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_1 2024-09-18T01:19:48.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:19:48.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:19:48.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:48.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:48.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:19:48.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:19:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_1 2024-09-18T01:19:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:19:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:19:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_1.status 2024-09-18T01:19:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_1.status 2024-09-18T01:19:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:19:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_1.status 2024-09-18T01:19:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_1.status 2024-09-18T01:19:48.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_1.status 2024-09-18T01:19:48.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:19:48.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.status 2024-09-18T01:19:48.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_1 2024-09-18T01:19:48.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:19:48.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:48.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:48.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_1 2024-09-18T01:19:48.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 86 2024-09-18T01:19:49.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_1 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_1.status 2024-09-18T01:19:49.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:19:49.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:49.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:49.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_1.status 2024-09-18T01:19:49.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_1 --format xml 2024-09-18T01:19:49.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:19:49.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:19:49.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=86 2024-09-18T01:19:49.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:19:49.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:19:49.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:19:49.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.status 2024-09-18T01:19:49.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:49.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:49.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:49.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.status 2024-09-18T01:19:49.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2024-09-18T01:19:49.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:19:49.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:19:49.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:19:49.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:19:49.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:19:49.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:19:49.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:19:49.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 86 = '' 2024-09-18T01:19:49.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:19:49.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:19:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.status 2024-09-18T01:19:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.status 2024-09-18T01:19:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2024-09-18T01:19:49.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:19:49.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:19:49.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:19:49.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:19:49.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:19:49.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:19:49.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:19:49.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 86 = '' 2024-09-18T01:19:49.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:19:49.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.status 2024-09-18T01:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:50.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:50.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:50.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.status 2024-09-18T01:19:50.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2024-09-18T01:19:50.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:19:50.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=86 2024-09-18T01:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 86 = 86 2024-09-18T01:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap 2024-09-18T01:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:19:50.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:19:51.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap 2024-09-18T01:19:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_1': 2024-09-18T01:19:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:19:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:19:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:19:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc967579fc 2024-09-18T01:19:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc967579fc 2024-09-18T01:19:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:19:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:19:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:19:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:19:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:09:21 2024 2024-09-18T01:19:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:09:21 2024 2024-09-18T01:19:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:09:21 2024 2024-09-18T01:19:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:19:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:19:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:19:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 1bfda626-752e-40be-8f8c-5733d4c53ac8 2024-09-18T01:19:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:19:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:19:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_1 snap 2024-09-18T01:19:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:19:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:19:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:19:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_1.export 2024-09-18T01:19:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.export 2024-09-18T01:19:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_1.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.export 2024-09-18T01:19:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_1.export 2024-09-18T01:19:51.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:19:51.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.export 2024-09-18T01:19:52.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:19:52.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_1.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.export 2024-09-18T01:19:52.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_1.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_1.export 2024-09-18T01:19:52.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:19:52.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:19:52.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_2 2024-09-18T01:19:52.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_2 snap 2024-09-18T01:19:52.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:19:52.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:52.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:52.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:19:52.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:19:54.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_2 2024-09-18T01:19:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started 2024-09-18T01:19:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:19:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:19:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:19:54.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:19:55.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_2 started 2024-09-18T01:19:55.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:55.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:55.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:55.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:19:55.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:19:55.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:19:55.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2 2024-09-18T01:19:55.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:19:55.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:19:55.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:55.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:19:55.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:19:55.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:19:55.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:19:55.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:19:55.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:19:55.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2 2024-09-18T01:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:19:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:55.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:19:55.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:19:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:19:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:19:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:19:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:19:55.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:19:55.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:55.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:55.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:19:55.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2 2024-09-18T01:19:55.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2024-09-18T01:19:55.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:19:55.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:55.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:55.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_2 2024-09-18T01:19:55.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:19:55.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:55.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:55.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:19:55.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:19:55.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:55.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:55.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:55.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:19:55.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:55.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:19:55.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:19:55.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:55.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:55.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:19:55.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:19:55.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:19:55.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:55.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:55.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:19:55.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_2 2024-09-18T01:19:55.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 5 closest matches: 2024-09-18T01:19:55.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_20 2024-09-18T01:19:55.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_21 2024-09-18T01:19:55.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_22 2024-09-18T01:19:55.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_23 2024-09-18T01:19:55.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_24 2024-09-18T01:19:55.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:19:55.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:19:55.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_2 2024-09-18T01:19:55.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:19:55.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:19:55.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:19:55.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:19:55.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:19:55.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:19:55.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:19:55.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:19:55.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:19:55.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:55.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:19:55.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:55.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:19:55.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:55.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:19:55.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:19:55.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:19:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:19:55.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:19:55.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:55.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:55.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:19:55.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_2 2024-09-18T01:19:55.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:19:55.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:19:55.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:19:55.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:19:55.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:19:55.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:19:55.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:19:55.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:19:55.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_2 2024-09-18T01:19:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:19:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:19:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:19:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:19:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_2 2024-09-18T01:19:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:19:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:19:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:55.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:55.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_2.status 2024-09-18T01:19:55.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_2.status 2024-09-18T01:19:55.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:19:55.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_2.status 2024-09-18T01:19:55.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_2.status 2024-09-18T01:19:55.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_2.status 2024-09-18T01:19:55.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:19:55.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.status 2024-09-18T01:19:55.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_2 2024-09-18T01:19:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:19:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_2 2024-09-18T01:19:56.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 88 2024-09-18T01:19:56.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_2 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_2.status 2024-09-18T01:19:56.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:19:56.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:56.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:56.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_2.status 2024-09-18T01:19:56.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_2 --format xml 2024-09-18T01:19:56.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:19:56.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:19:56.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=88 2024-09-18T01:19:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:19:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:19:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:19:56.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.status 2024-09-18T01:19:56.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:56.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:56.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:56.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.status 2024-09-18T01:19:56.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-09-18T01:19:56.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:19:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:19:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=36 2024-09-18T01:19:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 88 = 36 2024-09-18T01:19:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:19:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:19:56.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.status 2024-09-18T01:19:56.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:56.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:56.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:56.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.status 2024-09-18T01:19:56.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-09-18T01:19:56.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:19:56.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:19:56.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:19:56.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:19:56.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:19:56.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:19:56.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:19:56.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 88 = '' 2024-09-18T01:19:56.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:19:56.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:19:57.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.status 2024-09-18T01:19:57.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:57.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:57.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:57.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.status 2024-09-18T01:19:57.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-09-18T01:19:57.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:19:57.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:19:57.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:19:57.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:19:57.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:19:57.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:19:57.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:19:57.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 88 = '' 2024-09-18T01:19:57.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:19:57.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:19:59.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.status 2024-09-18T01:19:59.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:59.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:59.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:59.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.status 2024-09-18T01:19:59.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-09-18T01:19:59.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:19:59.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:19:59.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=88 2024-09-18T01:19:59.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 88 = 88 2024-09-18T01:19:59.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:19:59.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap 2024-09-18T01:19:59.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:19:59.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:19:59.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:19:59.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:19:59.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:19:59.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:19:59.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap 2024-09-18T01:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_2': 2024-09-18T01:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1140d657c95f 2024-09-18T01:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1140d657c95f 2024-09-18T01:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:20:00.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:20:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:20:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:20:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:09:29 2024 2024-09-18T01:20:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:09:29 2024 2024-09-18T01:20:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:09:29 2024 2024-09-18T01:20:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:20:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:20:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:20:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 0efe7b51-8742-400d-adca-bf1d51dea26b 2024-09-18T01:20:00.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:20:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_2 snap 2024-09-18T01:20:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:20:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:20:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_2.export 2024-09-18T01:20:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.export 2024-09-18T01:20:00.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_2.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.export 2024-09-18T01:20:00.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_2.export 2024-09-18T01:20:00.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:00.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.export 2024-09-18T01:20:00.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:00.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_2.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.export 2024-09-18T01:20:01.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_2.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_2.export 2024-09-18T01:20:01.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:20:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_3 2024-09-18T01:20:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_3 snap 2024-09-18T01:20:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:01.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:01.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:20:02.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_3 2024-09-18T01:20:02.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:02.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:02.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:02.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started 2024-09-18T01:20:02.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:02.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:02.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:02.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:20:02.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:02.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:02.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:03.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_3 started 2024-09-18T01:20:03.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:03.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:03.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:03.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:20:03.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:20:03.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:20:03.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3 2024-09-18T01:20:03.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:20:03.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:20:03.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:03.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:20:03.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:20:03.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:20:03.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:03.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:20:03.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:03.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:03.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:20:03.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:03.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:20:03.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:20:03.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:03.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:03.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:20:03.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:20:03.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:03.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:03.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:03.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:03.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:03.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:03.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:03.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:20:03.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:20:03.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:20:03.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:03.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:03.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:03.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3 2024-09-18T01:20:03.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2024-09-18T01:20:03.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:20:03.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:03.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:03.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_3 2024-09-18T01:20:03.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:20:03.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:20:03.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:20:03.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:20:03.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:03.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:03.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:03.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_3 2024-09-18T01:20:03.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 1 closest matches: 2024-09-18T01:20:03.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_33 2024-09-18T01:20:03.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_3 2024-09-18T01:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:03.703 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:03.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:20:03.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:20:03.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:03.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_3 2024-09-18T01:20:03.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:20:03.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:20:03.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:20:03.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:20:03.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:20:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:20:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:20:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_3 2024-09-18T01:20:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:20:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:20:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_3 2024-09-18T01:20:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_3.status 2024-09-18T01:20:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_3.status 2024-09-18T01:20:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:03.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_3.status 2024-09-18T01:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_3.status 2024-09-18T01:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_3.status 2024-09-18T01:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.status 2024-09-18T01:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_3 2024-09-18T01:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_3 2024-09-18T01:20:04.485 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 90 2024-09-18T01:20:04.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_3 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_3.status 2024-09-18T01:20:04.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:04.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:04.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:04.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_3.status 2024-09-18T01:20:04.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_3 --format xml 2024-09-18T01:20:04.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:04.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:20:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=90 2024-09-18T01:20:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:04.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:20:04.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.status 2024-09-18T01:20:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.status 2024-09-18T01:20:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2024-09-18T01:20:04.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:04.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:04.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:04.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:04.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:04.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:04.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:04.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 90 = '' 2024-09-18T01:20:04.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:04.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:20:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.status 2024-09-18T01:20:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.status 2024-09-18T01:20:05.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2024-09-18T01:20:05.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:05.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:05.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:05.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:05.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:05.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:05.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:05.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 90 = '' 2024-09-18T01:20:05.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:05.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:20:06.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.status 2024-09-18T01:20:06.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:06.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:06.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:06.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.status 2024-09-18T01:20:06.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2024-09-18T01:20:06.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:06.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:06.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=90 2024-09-18T01:20:06.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 90 = 90 2024-09-18T01:20:06.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:06.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap 2024-09-18T01:20:06.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:06.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:06.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:06.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:06.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:06.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:20:06.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:07.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap 2024-09-18T01:20:07.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_3': 2024-09-18T01:20:07.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:20:07.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:20:07.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:20:07.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1268de7a9d30 2024-09-18T01:20:07.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1268de7a9d30 2024-09-18T01:20:07.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:20:07.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:20:07.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:20:07.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:20:07.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:02 2024 2024-09-18T01:20:07.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:02 2024 2024-09-18T01:20:07.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:02 2024 2024-09-18T01:20:07.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:20:07.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:20:07.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:20:07.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 26ca4ff3-0678-4f67-8c75-7af181831497 2024-09-18T01:20:07.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:20:07.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:07.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_3 snap 2024-09-18T01:20:07.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:07.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:20:07.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:07.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:20:07.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_3.export 2024-09-18T01:20:07.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.export 2024-09-18T01:20:07.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_3.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.export 2024-09-18T01:20:07.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_3.export 2024-09-18T01:20:07.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:07.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.export 2024-09-18T01:20:07.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:07.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_3.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.export 2024-09-18T01:20:07.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_3.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_3.export 2024-09-18T01:20:07.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:07.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:20:07.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_4 2024-09-18T01:20:07.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_4 snap 2024-09-18T01:20:07.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:07.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:07.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:07.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:07.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:20:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_4 2024-09-18T01:20:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started 2024-09-18T01:20:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:20:09.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:09.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:09.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_4 started 2024-09-18T01:20:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:20:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:20:10.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:20:10.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4 2024-09-18T01:20:10.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:20:10.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:10.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:20:10.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:20:10.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:20:10.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:20:10.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:10.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:20:10.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:20:10.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:20:10.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:10.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2024-09-18T01:20:10.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:20:10.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:10.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:10.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:20:10.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:20:10.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:10.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:20:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:20:10.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:10.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:10.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:10.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:10.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:10.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:10.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:10.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:20:10.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:20:10.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:20:10.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:10.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:10.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:10.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2024-09-18T01:20:10.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 4 closest matches: 2024-09-18T01:20:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_44 2024-09-18T01:20:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_45 2024-09-18T01:20:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_46 2024-09-18T01:20:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_48 2024-09-18T01:20:10.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:10.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:10.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_4 2024-09-18T01:20:10.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:20:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:20:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:20:10.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:20:10.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:10.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_4 2024-09-18T01:20:10.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 2 closest matches: 2024-09-18T01:20:10.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_41 2024-09-18T01:20:10.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_49 2024-09-18T01:20:10.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_4 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:10.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:10.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:10.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:20:10.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:20:10.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:20:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:10.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_4 2024-09-18T01:20:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:no valid command found; 2 closest matches: 2024-09-18T01:20:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_43 2024-09-18T01:20:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status mirror/image_47 2024-09-18T01:20:10.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_4 2024-09-18T01:20:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:20:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:20:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:20:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:10.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:10.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:20:10.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:20:10.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:10.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:10.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:20:10.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:20:10.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:20:10.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:20:10.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:20:10.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:20:10.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_4 2024-09-18T01:20:10.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:20:10.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:20:10.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:20:10.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:20:10.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:20:10.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:20:10.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:20:10.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:10.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_4 2024-09-18T01:20:10.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_4 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_4.status 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_4.status 2024-09-18T01:20:10.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_4.status 2024-09-18T01:20:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_4.status 2024-09-18T01:20:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_4.status 2024-09-18T01:20:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.status 2024-09-18T01:20:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_4 2024-09-18T01:20:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:10.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_4 2024-09-18T01:20:11.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 92 2024-09-18T01:20:11.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_4 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_4.status 2024-09-18T01:20:11.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:11.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:11.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:11.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_4.status 2024-09-18T01:20:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_4 --format xml 2024-09-18T01:20:11.510 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:11.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=92 2024-09-18T01:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:11.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:20:11.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.status 2024-09-18T01:20:11.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:11.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:11.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:11.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.status 2024-09-18T01:20:11.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2024-09-18T01:20:11.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:11.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:11.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:11.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 92 = '' 2024-09-18T01:20:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:11.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:20:12.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.status 2024-09-18T01:20:12.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:12.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:12.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:12.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.status 2024-09-18T01:20:12.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2024-09-18T01:20:12.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:12.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:12.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:12.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:12.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:12.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:12.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:12.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 92 = '' 2024-09-18T01:20:12.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:12.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:20:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.status 2024-09-18T01:20:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.status 2024-09-18T01:20:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2024-09-18T01:20:13.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:13.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=92 2024-09-18T01:20:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 92 = 92 2024-09-18T01:20:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap 2024-09-18T01:20:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:13.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:13.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:13.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:13.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:20:13.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:14.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap 2024-09-18T01:20:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_4': 2024-09-18T01:20:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:20:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:20:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:20:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 11c3268a8f21 2024-09-18T01:20:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.11c3268a8f21 2024-09-18T01:20:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:20:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:20:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:20:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:20:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:09:56 2024 2024-09-18T01:20:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:09:56 2024 2024-09-18T01:20:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:09:56 2024 2024-09-18T01:20:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:20:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:20:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:20:14.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: f06096fd-c27d-4c66-a888-be876f867f5a 2024-09-18T01:20:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:20:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_4 snap 2024-09-18T01:20:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:20:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:20:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_4.export 2024-09-18T01:20:14.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.export 2024-09-18T01:20:14.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_4.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.export 2024-09-18T01:20:14.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_4.export 2024-09-18T01:20:14.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:14.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.export 2024-09-18T01:20:14.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:14.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_4.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.export 2024-09-18T01:20:14.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_4.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_4.export 2024-09-18T01:20:14.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:20:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_5 2024-09-18T01:20:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_5 snap 2024-09-18T01:20:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:20:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_5 2024-09-18T01:20:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started 2024-09-18T01:20:16.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:20:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:16.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:17.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_5 started 2024-09-18T01:20:17.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:17.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:17.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:17.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:20:17.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:20:17.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:20:17.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5 2024-09-18T01:20:17.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:20:17.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:17.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:20:17.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:20:17.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:20:17.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:20:17.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:17.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:20:17.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:20:17.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:20:17.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:17.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5 2024-09-18T01:20:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:20:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:20:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:20:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:20:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:17.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:17.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:20:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:20:17.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:20:17.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:17.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:17.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5 2024-09-18T01:20:17.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:20:17.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:20:17.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:20:17.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:20:17.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:20:17.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:20:17.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:20:17.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_5 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_5 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_5.status 2024-09-18T01:20:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_5.status 2024-09-18T01:20:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_5.status 2024-09-18T01:20:17.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_5.status 2024-09-18T01:20:17.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_5.status 2024-09-18T01:20:17.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:17.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.status 2024-09-18T01:20:17.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_5 2024-09-18T01:20:17.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:17.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:17.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:17.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_5 2024-09-18T01:20:18.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 94 2024-09-18T01:20:18.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_5 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_5.status 2024-09-18T01:20:18.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:18.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:18.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:18.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_5.status 2024-09-18T01:20:18.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_5 --format xml 2024-09-18T01:20:18.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:18.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:20:18.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=94 2024-09-18T01:20:18.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:18.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:18.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:20:18.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.status 2024-09-18T01:20:18.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:18.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:18.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:18.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.status 2024-09-18T01:20:18.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2024-09-18T01:20:18.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:18.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:18.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:18.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:18.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:18.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:18.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:18.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 94 = '' 2024-09-18T01:20:18.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:18.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:20:19.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.status 2024-09-18T01:20:19.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:19.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:19.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:19.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.status 2024-09-18T01:20:19.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2024-09-18T01:20:19.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:19.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:19.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 94 = '' 2024-09-18T01:20:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:20:20.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.status 2024-09-18T01:20:20.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:20.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:20.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:20.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.status 2024-09-18T01:20:20.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2024-09-18T01:20:20.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:20.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=94 2024-09-18T01:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 94 = 94 2024-09-18T01:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap 2024-09-18T01:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:20:20.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:21.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap 2024-09-18T01:20:21.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_5': 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc4df208de 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc4df208de 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:11 2024 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:11 2024 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:11 2024 2024-09-18T01:20:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:20:21.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:20:21.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:20:21.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: abb2637a-7d4a-4f6e-a9ca-ee069dffe536 2024-09-18T01:20:21.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:20:21.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:21.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_5 snap 2024-09-18T01:20:21.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:21.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:20:21.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:21.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:20:21.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_5.export 2024-09-18T01:20:21.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.export 2024-09-18T01:20:21.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_5.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.export 2024-09-18T01:20:21.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_5.export 2024-09-18T01:20:21.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:21.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.export 2024-09-18T01:20:21.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:21.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_5.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.export 2024-09-18T01:20:21.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_5.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_5.export 2024-09-18T01:20:21.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:21.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:20:21.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_6 2024-09-18T01:20:21.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_6 snap 2024-09-18T01:20:21.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:21.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:21.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:21.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:21.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:20:23.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_6 2024-09-18T01:20:23.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:23.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:23.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:23.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started 2024-09-18T01:20:23.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:23.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:23.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:23.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:20:23.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:23.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:23.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:24.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_6 started 2024-09-18T01:20:24.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:24.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:24.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:24.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:20:24.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:20:24.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:20:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6 2024-09-18T01:20:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:20:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:20:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:20:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:20:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:20:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:24.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:20:24.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:20:24.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:20:24.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6 2024-09-18T01:20:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:20:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:20:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:20:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:20:24.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:20:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:20:24.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:20:24.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:24.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:24.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:20:24.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:20:24.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:24.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:24.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:24.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:24.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:24.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:24.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:24.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:20:24.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:20:24.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:20:24.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:24.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:24.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:24.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6 2024-09-18T01:20:24.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2024-09-18T01:20:24.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:20:24.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:24.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:24.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_6 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:24.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:20:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:24.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:24.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:20:24.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:20:24.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:20:24.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:24.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:24.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:24.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_6 2024-09-18T01:20:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:20:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:20:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:20:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:20:24.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:20:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:20:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:20:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_6 2024-09-18T01:20:24.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_6 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_6.status 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_6.status 2024-09-18T01:20:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_6.status 2024-09-18T01:20:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_6.status 2024-09-18T01:20:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_6.status 2024-09-18T01:20:24.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.status 2024-09-18T01:20:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_6 2024-09-18T01:20:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_6 2024-09-18T01:20:25.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 96 2024-09-18T01:20:25.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_6 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_6.status 2024-09-18T01:20:25.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:25.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:25.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:25.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_6.status 2024-09-18T01:20:25.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_6 --format xml 2024-09-18T01:20:25.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:25.587 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:20:25.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=96 2024-09-18T01:20:25.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:25.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:25.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:20:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.status 2024-09-18T01:20:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:25.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:25.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.status 2024-09-18T01:20:25.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2024-09-18T01:20:25.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:25.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:25.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:25.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:25.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:25.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:25.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:25.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 96 = '' 2024-09-18T01:20:25.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:25.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:20:26.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.status 2024-09-18T01:20:26.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:26.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:26.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:26.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.status 2024-09-18T01:20:26.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2024-09-18T01:20:26.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:26.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:26.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:26.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:26.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:26.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:26.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 96 = '' 2024-09-18T01:20:26.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:26.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:20:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.status 2024-09-18T01:20:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.status 2024-09-18T01:20:27.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2024-09-18T01:20:27.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:27.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=96 2024-09-18T01:20:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 96 = 96 2024-09-18T01:20:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap 2024-09-18T01:20:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:20:27.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:27.454 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.3774892814030657) > weight (0.25) 2024-09-18T01:20:27.454 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.827658959577119) > weight (0.25) 2024-09-18T01:20:27.454 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5187184624430551) > weight (0.25) 2024-09-18T01:20:27.454 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6367548651542685) > weight (0.25) 2024-09-18T01:20:27.455 INFO:tasks.rbd_mirror_thrash:waiting for 94 secs before thrashing 2024-09-18T01:20:28.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap 2024-09-18T01:20:28.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_6': 2024-09-18T01:20:28.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:20:28.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1140ee2b7254 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1140ee2b7254 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:20 2024 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:20 2024 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:20 2024 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:20:28.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:20:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:20:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: cbcb3575-14f3-490d-a7c4-517bdeb6f42a 2024-09-18T01:20:28.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:20:28.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:28.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_6 snap 2024-09-18T01:20:28.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:28.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:20:28.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:20:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_6.export 2024-09-18T01:20:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.export 2024-09-18T01:20:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_6.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.export 2024-09-18T01:20:28.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_6.export 2024-09-18T01:20:28.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:28.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.export 2024-09-18T01:20:28.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:28.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_6.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.export 2024-09-18T01:20:28.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_6.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_6.export 2024-09-18T01:20:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:20:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_7 2024-09-18T01:20:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_7 snap 2024-09-18T01:20:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:28.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:20:30.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_7 2024-09-18T01:20:30.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:30.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:30.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:30.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started 2024-09-18T01:20:30.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:30.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:30.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:30.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:20:30.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:30.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:30.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:31.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_7 started 2024-09-18T01:20:31.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:31.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:31.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:31.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:20:31.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:20:31.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:20:31.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7 2024-09-18T01:20:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:20:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:20:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:20:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:20:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:20:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:20:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:20:31.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:20:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7 2024-09-18T01:20:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:20:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:20:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:20:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:31.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:20:31.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:31.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:31.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:31.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:31.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:31.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:31.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:31.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:20:31.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:20:31.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:20:31.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:31.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:31.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:31.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7 2024-09-18T01:20:31.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2024-09-18T01:20:31.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:20:31.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_7 2024-09-18T01:20:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:20:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:20:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:20:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:20:31.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:31.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:31.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:31.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:20:31.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:20:31.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:31.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:31.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:20:31.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:20:31.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:20:31.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:31.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:31.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:31.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_7 2024-09-18T01:20:31.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2024-09-18T01:20:31.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:20:31.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_7 2024-09-18T01:20:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:20:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:31.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:20:31.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:20:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:20:31.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:31.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:31.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:31.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_7 2024-09-18T01:20:31.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2024-09-18T01:20:31.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:20:31.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_7 2024-09-18T01:20:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:20:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:20:31.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:20:31.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:20:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:20:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:20:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:20:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:20:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:20:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:20:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:20:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_7 2024-09-18T01:20:31.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:20:31.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:20:31.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:20:31.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:20:31.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:20:31.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:20:31.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:20:31.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:31.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_7 2024-09-18T01:20:31.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_7 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_7.status 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_7.status 2024-09-18T01:20:31.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:31.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_7.status 2024-09-18T01:20:32.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_7.status 2024-09-18T01:20:32.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_7.status 2024-09-18T01:20:32.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:32.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.status 2024-09-18T01:20:32.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_7 2024-09-18T01:20:32.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:32.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:32.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:32.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_7 2024-09-18T01:20:32.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 98 2024-09-18T01:20:32.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_7 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_7.status 2024-09-18T01:20:32.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:32.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:32.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:32.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_7.status 2024-09-18T01:20:32.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_7 --format xml 2024-09-18T01:20:32.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:32.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:20:32.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=98 2024-09-18T01:20:32.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:32.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:32.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:20:32.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.status 2024-09-18T01:20:32.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:32.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:32.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:32.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.status 2024-09-18T01:20:32.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2024-09-18T01:20:32.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:32.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:32.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:32.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:32.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:32.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:32.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:32.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 98 = '' 2024-09-18T01:20:32.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:32.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:20:33.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.status 2024-09-18T01:20:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.status 2024-09-18T01:20:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2024-09-18T01:20:33.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:33.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:33.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:33.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:33.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:33.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:33.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:33.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 98 = '' 2024-09-18T01:20:33.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:33.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:20:34.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.status 2024-09-18T01:20:34.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:34.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:34.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:34.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.status 2024-09-18T01:20:34.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2024-09-18T01:20:34.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:34.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:34.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=98 2024-09-18T01:20:34.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 98 = 98 2024-09-18T01:20:34.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:34.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap 2024-09-18T01:20:34.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:34.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:34.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:34.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:34.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:34.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:20:34.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:35.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap 2024-09-18T01:20:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_7': 2024-09-18T01:20:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:20:35.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 11c3c5c26df5 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.11c3c5c26df5 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:28 2024 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:28 2024 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:28 2024 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:20:35.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 60a348dd-1555-4a46-89d2-8585c357735d 2024-09-18T01:20:35.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:20:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_7 snap 2024-09-18T01:20:35.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:20:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:20:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_7.export 2024-09-18T01:20:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.export 2024-09-18T01:20:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_7.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.export 2024-09-18T01:20:35.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_7.export 2024-09-18T01:20:35.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:35.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.export 2024-09-18T01:20:35.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:35.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_7.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.export 2024-09-18T01:20:35.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_7.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_7.export 2024-09-18T01:20:35.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:35.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:20:35.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_8 2024-09-18T01:20:35.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_8 snap 2024-09-18T01:20:35.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:35.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:35.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:35.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:35.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:20:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_8 2024-09-18T01:20:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started 2024-09-18T01:20:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:37.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:37.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:20:37.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:37.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:37.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:38.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_8 started 2024-09-18T01:20:38.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:38.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:38.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:38.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:20:38.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:20:38.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:20:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8 2024-09-18T01:20:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:20:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:20:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:20:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:20:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:20:38.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:38.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:20:38.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:20:38.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:20:38.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:38.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8 2024-09-18T01:20:38.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:20:38.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:38.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:38.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:20:38.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:20:38.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:38.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:38.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:20:38.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:20:38.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:38.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:38.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:38.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:38.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:38.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:38.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:38.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:20:38.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:20:38.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:20:38.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:38.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:38.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:38.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8 2024-09-18T01:20:38.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:20:38.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:20:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:20:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:20:38.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:20:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:20:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:20:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_8 2024-09-18T01:20:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:38.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:20:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:20:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_8 2024-09-18T01:20:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_8.status 2024-09-18T01:20:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_8.status 2024-09-18T01:20:38.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:38.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_8.status 2024-09-18T01:20:38.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_8.status 2024-09-18T01:20:38.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_8.status 2024-09-18T01:20:38.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:38.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.status 2024-09-18T01:20:38.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_8 2024-09-18T01:20:38.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:38.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:38.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:38.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_8 2024-09-18T01:20:39.516 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-18T01:20:39.517 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-18T01:20:39.517 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-18T01:20:39.517 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-18T01:20:39.517 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:20:39.518 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:20:39.517+0000 7f295a90a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 48231) UID: 1000 2024-09-18T01:20:39.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 100 2024-09-18T01:20:39.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_8 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_8.status 2024-09-18T01:20:39.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:39.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:39.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:39.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_8.status 2024-09-18T01:20:39.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_8 --format xml 2024-09-18T01:20:39.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:39.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:20:39.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=100 2024-09-18T01:20:39.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:39.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:39.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:20:39.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.status 2024-09-18T01:20:39.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:39.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:39.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:39.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.status 2024-09-18T01:20:39.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-18T01:20:39.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:39.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:39.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:39.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:39.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 100 = '' 2024-09-18T01:20:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:39.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:20:40.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.status 2024-09-18T01:20:40.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:40.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:40.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:40.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.status 2024-09-18T01:20:40.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-18T01:20:40.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:40.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:40.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:40.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:40.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:40.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:40.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:40.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 100 = '' 2024-09-18T01:20:40.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:40.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:20:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.status 2024-09-18T01:20:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.status 2024-09-18T01:20:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-09-18T01:20:41.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:41.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:41.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=100 2024-09-18T01:20:41.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 100 = 100 2024-09-18T01:20:41.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:41.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap 2024-09-18T01:20:41.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:41.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:41.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:41.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:41.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:41.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:20:41.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:42.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap 2024-09-18T01:20:42.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_8': 2024-09-18T01:20:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:20:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:20:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:20:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1268eb2b85ad 2024-09-18T01:20:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1268eb2b85ad 2024-09-18T01:20:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:20:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:20:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:20:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:20:42.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:40 2024 2024-09-18T01:20:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:40 2024 2024-09-18T01:20:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:40 2024 2024-09-18T01:20:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:20:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:20:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:20:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: c8657da4-764c-45a1-9898-eb5d4ab62f4f 2024-09-18T01:20:42.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:20:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_8 snap 2024-09-18T01:20:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:20:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:20:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_8.export 2024-09-18T01:20:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.export 2024-09-18T01:20:42.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_8.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.export 2024-09-18T01:20:42.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_8.export 2024-09-18T01:20:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:42.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.export 2024-09-18T01:20:42.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:42.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_8.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.export 2024-09-18T01:20:42.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_8.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_8.export 2024-09-18T01:20:42.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:42.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:20:42.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_9 2024-09-18T01:20:42.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_9 snap 2024-09-18T01:20:42.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:42.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:42.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:42.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:42.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:20:44.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_9 2024-09-18T01:20:44.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:44.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:44.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:44.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started 2024-09-18T01:20:44.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:44.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:44.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:44.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:20:44.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:44.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:44.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_9 started 2024-09-18T01:20:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:45.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:20:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:20:45.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:20:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9 2024-09-18T01:20:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:20:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:20:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:20:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:20:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:20:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:45.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:20:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:20:45.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:20:45.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:45.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:45.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:20:45.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:45.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:20:45.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:20:45.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:45.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:45.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:20:45.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:20:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:20:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:20:45.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:20:45.619 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-18T01:20:45.619 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-18T01:20:45.619 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-18T01:20:45.622 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-18T01:20:45.622 INFO:tasks.rbd_mirror_thrash:waiting for 99 secs before thrashing 2024-09-18T01:20:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:45.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9 2024-09-18T01:20:45.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:20:45.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:20:45.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:20:45.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:20:45.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:20:45.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:20:45.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:20:45.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:45.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_9 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_9 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_9.status 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_9.status 2024-09-18T01:20:45.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:45.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_9.status 2024-09-18T01:20:45.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_9.status 2024-09-18T01:20:45.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_9.status 2024-09-18T01:20:45.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:45.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.status 2024-09-18T01:20:45.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_9 2024-09-18T01:20:45.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:45.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:45.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:45.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_9 2024-09-18T01:20:46.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 102 2024-09-18T01:20:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_9 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_9.status 2024-09-18T01:20:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_9.status 2024-09-18T01:20:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_9 --format xml 2024-09-18T01:20:46.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:46.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:20:46.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=102 2024-09-18T01:20:46.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:46.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:46.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:20:46.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.status 2024-09-18T01:20:46.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:46.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:46.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:46.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.status 2024-09-18T01:20:46.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2024-09-18T01:20:46.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:46.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:46.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:46.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:46.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:46.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:46.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:46.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 102 = '' 2024-09-18T01:20:46.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:46.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:20:47.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.status 2024-09-18T01:20:47.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:47.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:47.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:47.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.status 2024-09-18T01:20:47.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2024-09-18T01:20:47.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:47.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:47.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:47.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:47.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:47.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:47.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:47.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 102 = '' 2024-09-18T01:20:47.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:47.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:20:48.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.status 2024-09-18T01:20:48.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:48.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:48.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:48.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.status 2024-09-18T01:20:48.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2024-09-18T01:20:48.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:48.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=102 2024-09-18T01:20:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 102 = 102 2024-09-18T01:20:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap 2024-09-18T01:20:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:20:48.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:49.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap 2024-09-18T01:20:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_9': 2024-09-18T01:20:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:20:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:20:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:20:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bcb18392ad 2024-09-18T01:20:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bcb18392ad 2024-09-18T01:20:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:20:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:20:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:20:49.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:20:49.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:48 2024 2024-09-18T01:20:49.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:48 2024 2024-09-18T01:20:49.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:48 2024 2024-09-18T01:20:49.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:20:49.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:20:49.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:20:49.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 28c48b31-f556-4f13-90b4-b7bc04a088dd 2024-09-18T01:20:49.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:20:49.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:49.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_9 snap 2024-09-18T01:20:49.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:49.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:20:49.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:49.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:20:49.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_9.export 2024-09-18T01:20:49.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.export 2024-09-18T01:20:49.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_9.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.export 2024-09-18T01:20:49.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_9.export 2024-09-18T01:20:49.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:49.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.export 2024-09-18T01:20:49.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:49.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_9.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.export 2024-09-18T01:20:49.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_9.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_9.export 2024-09-18T01:20:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:20:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_10 2024-09-18T01:20:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_10 snap 2024-09-18T01:20:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:49.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:20:51.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_10 2024-09-18T01:20:51.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:51.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started 2024-09-18T01:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:51.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:52.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_10 started 2024-09-18T01:20:52.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:52.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:52.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:52.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:20:52.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:20:52.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:20:52.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10 2024-09-18T01:20:52.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:20:52.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:20:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:20:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:20:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:20:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:52.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:20:52.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:20:52.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:20:52.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:52.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2024-09-18T01:20:52.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:20:52.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:52.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:52.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:20:52.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:20:52.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:52.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:52.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:52.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:20:52.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:52.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:52.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:52.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:20:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:20:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:20:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:20:52.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:20:52.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:52.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:52.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:52.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:52.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:52.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:20:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:20:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:52.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2024-09-18T01:20:52.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2024-09-18T01:20:52.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:20:52.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:20:52.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:52.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:20:52.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:20:52.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:52.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:52.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:20:52.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:20:52.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:20:52.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:52.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:52.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:20:52.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2024-09-18T01:20:52.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2024-09-18T01:20:52.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:20:52.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:52.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:52.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_10 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:20:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:20:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:20:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:20:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:20:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:20:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:52.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:20:52.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:20:52.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:52.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:52.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:20:52.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:20:52.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:20:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:20:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_10 2024-09-18T01:20:52.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2024-09-18T01:20:52.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:20:52.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:20:52.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:52.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_10 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:20:52.952 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:20:52.953 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:20:52.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:20:52.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:20:52.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:20:52.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:20:52.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_10 2024-09-18T01:20:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:20:53.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:20:53.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:20:53.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:20:53.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:20:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:20:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:20:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_10 2024-09-18T01:20:53.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_10 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_10.status 2024-09-18T01:20:53.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_10.status 2024-09-18T01:20:53.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:53.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_10.status 2024-09-18T01:20:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_10.status 2024-09-18T01:20:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_10.status 2024-09-18T01:20:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.status 2024-09-18T01:20:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_10 2024-09-18T01:20:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_10 2024-09-18T01:20:53.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 104 2024-09-18T01:20:53.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_10 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_10.status 2024-09-18T01:20:53.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:53.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:53.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:53.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_10.status 2024-09-18T01:20:53.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_10 --format xml 2024-09-18T01:20:53.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:53.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:20:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=104 2024-09-18T01:20:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:20:53.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.status 2024-09-18T01:20:53.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:53.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:53.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:53.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.status 2024-09-18T01:20:53.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2024-09-18T01:20:53.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:53.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:53.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:53.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 104 = '' 2024-09-18T01:20:53.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:53.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:20:54.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.status 2024-09-18T01:20:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.status 2024-09-18T01:20:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2024-09-18T01:20:54.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:54.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:20:54.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:54.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:20:54.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:20:54.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:20:54.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:20:54.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 104 = '' 2024-09-18T01:20:54.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:20:54.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:20:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.status 2024-09-18T01:20:55.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:55.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:55.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:55.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.status 2024-09-18T01:20:55.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2024-09-18T01:20:55.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:20:55.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:20:55.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=104 2024-09-18T01:20:55.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 104 = 104 2024-09-18T01:20:55.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:55.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap 2024-09-18T01:20:55.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:55.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:55.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:55.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:55.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:55.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:20:55.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:56.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap 2024-09-18T01:20:56.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_10': 2024-09-18T01:20:56.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1140695e827 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1140695e827 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:10:59 2024 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:10:59 2024 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:10:59 2024 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:20:56.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:20:56.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:20:56.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 0a5efa51-897f-43ff-9dc4-ca67a0c87f5d 2024-09-18T01:20:56.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:20:56.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:56.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_10 snap 2024-09-18T01:20:56.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:56.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:20:56.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:56.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:20:56.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_10.export 2024-09-18T01:20:56.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.export 2024-09-18T01:20:56.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_10.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.export 2024-09-18T01:20:56.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_10.export 2024-09-18T01:20:56.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:56.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.export 2024-09-18T01:20:56.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:20:56.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_10.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.export 2024-09-18T01:20:56.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_10.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_10.export 2024-09-18T01:20:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:20:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_11 2024-09-18T01:20:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_11 snap 2024-09-18T01:20:56.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:20:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:20:56.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:20:58.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_11 2024-09-18T01:20:58.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:58.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:58.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:20:58.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started 2024-09-18T01:20:58.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:58.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:58.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:20:58.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:20:58.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:20:58.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:58.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:20:59.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_11 started 2024-09-18T01:20:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:20:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:20:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:20:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:20:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:20:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11 2024-09-18T01:20:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:20:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:20:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:20:59.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:20:59.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:20:59.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:20:59.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:20:59.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:20:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:20:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:20:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:20:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:20:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:20:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:20:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:20:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:20:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:20:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:20:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:20:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:20:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:20:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:20:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:20:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:20:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:20:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:20:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2024-09-18T01:20:59.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:20:59.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:20:59.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:20:59.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:20:59.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:20:59.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:20:59.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:20:59.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:20:59.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_11 2024-09-18T01:20:59.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_11 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_11.status 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_11.status 2024-09-18T01:20:59.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_11.status 2024-09-18T01:20:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_11.status 2024-09-18T01:20:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_11.status 2024-09-18T01:20:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:20:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.status 2024-09-18T01:20:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_11 2024-09-18T01:20:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:20:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:20:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:20:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_11 2024-09-18T01:21:00.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 106 2024-09-18T01:21:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_11 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_11.status 2024-09-18T01:21:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:21:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_11.status 2024-09-18T01:21:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_11 --format xml 2024-09-18T01:21:00.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:00.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:21:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=106 2024-09-18T01:21:00.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:00.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:00.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:21:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.status 2024-09-18T01:21:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:21:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.status 2024-09-18T01:21:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2024-09-18T01:21:00.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:00.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:00.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:00.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:00.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:00.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:00.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:00.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 106 = '' 2024-09-18T01:21:00.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:00.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:21:01.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.status 2024-09-18T01:21:01.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:01.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:01.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:21:01.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.status 2024-09-18T01:21:01.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2024-09-18T01:21:01.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:01.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:01.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:01.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:01.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:01.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:01.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:01.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 106 = '' 2024-09-18T01:21:01.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:01.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:21:02.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.status 2024-09-18T01:21:02.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:02.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:02.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:21:02.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.status 2024-09-18T01:21:02.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2024-09-18T01:21:02.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:02.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:02.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=106 2024-09-18T01:21:02.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 106 = 106 2024-09-18T01:21:02.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:02.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap 2024-09-18T01:21:02.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:21:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:21:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:03.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap 2024-09-18T01:21:03.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_11': 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 11c363448e5e 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.11c363448e5e 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:11:14 2024 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:11:14 2024 2024-09-18T01:21:03.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:11:14 2024 2024-09-18T01:21:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:21:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:21:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:21:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 067a8b9f-a615-4968-8d24-956a8b2892a6 2024-09-18T01:21:03.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:21:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_11 snap 2024-09-18T01:21:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:21:03.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:03.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:21:03.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_11.export 2024-09-18T01:21:03.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.export 2024-09-18T01:21:03.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_11.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.export 2024-09-18T01:21:03.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_11.export 2024-09-18T01:21:03.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:03.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.export 2024-09-18T01:21:03.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:03.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_11.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.export 2024-09-18T01:21:03.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_11.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_11.export 2024-09-18T01:21:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:21:03.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_12 2024-09-18T01:21:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_12 snap 2024-09-18T01:21:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:03.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:03.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:21:05.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_12 2024-09-18T01:21:05.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:05.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:05.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:05.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started 2024-09-18T01:21:05.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:05.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:05.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:05.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:21:05.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:05.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:05.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:06.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_12 started 2024-09-18T01:21:06.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:06.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:06.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:06.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:21:06.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:21:06.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:21:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12 2024-09-18T01:21:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:21:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:21:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:21:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:21:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:21:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:06.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:21:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:21:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:21:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2024-09-18T01:21:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:21:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:21:06.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:21:06.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:06.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:06.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:21:06.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:21:06.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:06.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:06.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:06.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:06.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:06.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:06.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:06.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:21:06.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:21:06.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:21:06.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:06.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:06.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:06.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2024-09-18T01:21:06.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:21:06.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:21:06.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:21:06.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:21:06.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:21:06.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:21:06.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:21:06.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:06.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_12 2024-09-18T01:21:06.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:06.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:06.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:06.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:06.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:21:06.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:21:06.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_12 2024-09-18T01:21:06.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:06.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:06.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:06.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:06.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_12.status 2024-09-18T01:21:06.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_12.status 2024-09-18T01:21:06.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:06.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_12.status 2024-09-18T01:21:06.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_12.status 2024-09-18T01:21:06.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_12.status 2024-09-18T01:21:06.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:06.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.status 2024-09-18T01:21:06.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_12 2024-09-18T01:21:06.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:06.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:06.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:06.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_12 2024-09-18T01:21:07.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 108 2024-09-18T01:21:07.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_12 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_12.status 2024-09-18T01:21:07.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:07.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:07.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:07.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_12.status 2024-09-18T01:21:07.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_12 --format xml 2024-09-18T01:21:07.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:07.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:21:07.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=108 2024-09-18T01:21:07.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:07.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:07.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:21:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.status 2024-09-18T01:21:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.status 2024-09-18T01:21:07.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2024-09-18T01:21:07.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:07.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:07.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:07.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:07.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:07.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:07.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:07.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 108 = '' 2024-09-18T01:21:07.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:07.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:21:08.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.status 2024-09-18T01:21:08.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:08.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:08.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:08.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.status 2024-09-18T01:21:08.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2024-09-18T01:21:08.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:08.423 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:08.423 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:08.424 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:08.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:08.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:08.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:08.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 108 = '' 2024-09-18T01:21:08.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:08.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:21:09.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.status 2024-09-18T01:21:09.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:09.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:09.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:09.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.status 2024-09-18T01:21:09.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2024-09-18T01:21:09.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:09.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=108 2024-09-18T01:21:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 108 = 108 2024-09-18T01:21:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:09.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap 2024-09-18T01:21:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:21:09.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:10.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap 2024-09-18T01:21:10.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_12': 2024-09-18T01:21:10.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:21:10.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:21:10.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1268708e4746 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1268708e4746 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:11:25 2024 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:11:25 2024 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:11:25 2024 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:21:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: b8fdbcd4-3f49-41c6-90b8-318c1f5ef36b 2024-09-18T01:21:10.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:21:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_12 snap 2024-09-18T01:21:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:10.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:21:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:21:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_12.export 2024-09-18T01:21:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.export 2024-09-18T01:21:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_12.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.export 2024-09-18T01:21:10.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_12.export 2024-09-18T01:21:10.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:10.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.export 2024-09-18T01:21:10.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:10.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_12.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.export 2024-09-18T01:21:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_12.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_12.export 2024-09-18T01:21:10.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:10.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:21:10.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_13 2024-09-18T01:21:10.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_13 snap 2024-09-18T01:21:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:10.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_13 2024-09-18T01:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started 2024-09-18T01:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:12.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:12.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_13 started 2024-09-18T01:21:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:21:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:21:13.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:21:13.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13 2024-09-18T01:21:13.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:21:13.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:21:13.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:13.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:21:13.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:21:13.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:21:13.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:13.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:21:13.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:21:13.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:13.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:13.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:13.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:13.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:13.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:21:13.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:21:13.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:21:13.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:13.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:13.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:13.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2024-09-18T01:21:13.797 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:21:13.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:21:13.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:21:13.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:21:13.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:21:13.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:21:13.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:21:13.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:13.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_13 2024-09-18T01:21:13.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_13 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_13.status 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_13.status 2024-09-18T01:21:13.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:13.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_13.status 2024-09-18T01:21:13.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_13.status 2024-09-18T01:21:13.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_13.status 2024-09-18T01:21:13.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:13.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.status 2024-09-18T01:21:13.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_13 2024-09-18T01:21:13.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:13.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:13.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:13.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_13 2024-09-18T01:21:14.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 110 2024-09-18T01:21:14.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_13 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_13.status 2024-09-18T01:21:14.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:14.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_13.status 2024-09-18T01:21:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_13 --format xml 2024-09-18T01:21:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:14.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:21:14.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=110 2024-09-18T01:21:14.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:14.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:14.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:21:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.status 2024-09-18T01:21:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.status 2024-09-18T01:21:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2024-09-18T01:21:14.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:15.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:15.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:15.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:15.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:15.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:15.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:15.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 110 = '' 2024-09-18T01:21:15.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:15.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:21:15.406 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.status 2024-09-18T01:21:15.406 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:15.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:15.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:15.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.status 2024-09-18T01:21:15.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2024-09-18T01:21:15.407 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:15.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:15.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:15.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:15.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:15.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 110 = '' 2024-09-18T01:21:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:15.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:21:16.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.status 2024-09-18T01:21:16.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:16.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:16.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:16.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.status 2024-09-18T01:21:16.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2024-09-18T01:21:16.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:16.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=110 2024-09-18T01:21:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 110 = 110 2024-09-18T01:21:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap 2024-09-18T01:21:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:21:16.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:17.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap 2024-09-18T01:21:17.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_13': 2024-09-18T01:21:17.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:21:17.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:21:17.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:21:17.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc15a5a911 2024-09-18T01:21:17.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc15a5a911 2024-09-18T01:21:17.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:21:17.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:21:17.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:21:17.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:21:17.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:11:37 2024 2024-09-18T01:21:17.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:11:37 2024 2024-09-18T01:21:17.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:11:37 2024 2024-09-18T01:21:17.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:21:17.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:21:17.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:21:17.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: a60dcf1c-e06a-4d99-b676-d35f298225a4 2024-09-18T01:21:17.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:21:17.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:17.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_13 snap 2024-09-18T01:21:17.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:17.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:21:17.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:17.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:21:17.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_13.export 2024-09-18T01:21:17.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.export 2024-09-18T01:21:17.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_13.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.export 2024-09-18T01:21:17.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_13.export 2024-09-18T01:21:17.539 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.export 2024-09-18T01:21:17.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:17.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_13.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.export 2024-09-18T01:21:17.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_13.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_13.export 2024-09-18T01:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_14 2024-09-18T01:21:17.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_14 snap 2024-09-18T01:21:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:21:19.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_14 2024-09-18T01:21:19.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:19.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:19.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:19.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started 2024-09-18T01:21:19.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:19.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:19.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:19.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:21:19.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:19.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:19.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:20.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_14 started 2024-09-18T01:21:20.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:20.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:20.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:20.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:21:20.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:21:20.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:21:20.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14 2024-09-18T01:21:20.708 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:21:20.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:21:20.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:20.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:21:20.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:21:20.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:21:20.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:20.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14 2024-09-18T01:21:20.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:21:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:21:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:21:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:20.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:20.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:20.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:21:20.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:20.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:20.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:20.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:21:20.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:20.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:21:20.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:21:20.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:20.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:20.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:21:20.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:21:20.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:20.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:20.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:20.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:20.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:20.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:20.721 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:20.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:21:20.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:21:20.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:21:20.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:20.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:20.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:20.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14 2024-09-18T01:21:20.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-18T01:21:20.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:21:20.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14 2024-09-18T01:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:20.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:20.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:20.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:20.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:21:20.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:21:20.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:20.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:20.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:21:20.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:21:20.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:21:20.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:20.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:20.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:20.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14 2024-09-18T01:21:20.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-18T01:21:20.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:21:20.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:21:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_14 2024-09-18T01:21:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:21:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:21:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:21:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:20.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:21:20.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:21:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:21:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:21:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:21:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:21:20.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:21:20.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:21:20.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:21:20.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:21:20.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_14 2024-09-18T01:21:21.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-09-18T01:21:21.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:21:21.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:21:21.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_14 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:21:21.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:21:21.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:21:21.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:21.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:21.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:21:21.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:21:21.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:21:21.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:21:21.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:21:21.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:21:21.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_14 2024-09-18T01:21:21.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:21:21.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:21:21.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:21:21.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:21:21.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:21:21.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:21:21.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:21:21.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:21.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_14 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_14 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_14.status 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_14.status 2024-09-18T01:21:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:21.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_14.status 2024-09-18T01:21:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_14.status 2024-09-18T01:21:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_14.status 2024-09-18T01:21:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.status 2024-09-18T01:21:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_14 2024-09-18T01:21:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:21.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_14 2024-09-18T01:21:21.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 112 2024-09-18T01:21:21.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_14 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_14.status 2024-09-18T01:21:21.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:21.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:21.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:21.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_14.status 2024-09-18T01:21:21.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_14 --format xml 2024-09-18T01:21:21.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:21.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:21:21.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=112 2024-09-18T01:21:21.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:21.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:21.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:21:21.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.status 2024-09-18T01:21:21.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:21.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:21.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:21.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.status 2024-09-18T01:21:21.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2024-09-18T01:21:21.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:22.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:22.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:22.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:22.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 112 = '' 2024-09-18T01:21:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:22.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:21:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.status 2024-09-18T01:21:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.status 2024-09-18T01:21:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2024-09-18T01:21:22.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:22.458 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:22.458 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:22.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:22.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 112 = '' 2024-09-18T01:21:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:22.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:21:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.status 2024-09-18T01:21:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.status 2024-09-18T01:21:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2024-09-18T01:21:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:23.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=112 2024-09-18T01:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 112 = 112 2024-09-18T01:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap 2024-09-18T01:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:24.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_14': 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1140da238a90 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1140da238a90 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:11:43 2024 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:11:43 2024 2024-09-18T01:21:24.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:11:43 2024 2024-09-18T01:21:24.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:21:24.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:21:24.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:21:24.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 6c6c64a2-50e5-44f2-8703-2da1463fc540 2024-09-18T01:21:24.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:21:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_14 snap 2024-09-18T01:21:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:21:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:21:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_14.export 2024-09-18T01:21:24.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.export 2024-09-18T01:21:24.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_14.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.export 2024-09-18T01:21:24.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_14.export 2024-09-18T01:21:24.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.export 2024-09-18T01:21:24.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:24.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_14.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.export 2024-09-18T01:21:24.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_14.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_14.export 2024-09-18T01:21:24.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:24.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:21:24.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_15 2024-09-18T01:21:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_15 snap 2024-09-18T01:21:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:21:26.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_15 2024-09-18T01:21:26.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:26.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started 2024-09-18T01:21:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:21:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:26.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_15 started 2024-09-18T01:21:27.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:21:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:21:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:21:27.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15 2024-09-18T01:21:27.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:21:27.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:27.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:21:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:21:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:21:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:21:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:21:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:21:27.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:21:27.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:27.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15 2024-09-18T01:21:27.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:21:27.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:27.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:27.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:21:27.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:21:27.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:21:27.724 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:21:27.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:27.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:27.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:27.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15 2024-09-18T01:21:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:21:27.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:21:27.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:21:27.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:21:27.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:21:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:21:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:21:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_15 2024-09-18T01:21:27.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_15 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_15.status 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_15.status 2024-09-18T01:21:27.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:27.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_15.status 2024-09-18T01:21:27.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_15.status 2024-09-18T01:21:27.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_15.status 2024-09-18T01:21:27.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:27.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.status 2024-09-18T01:21:27.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_15 2024-09-18T01:21:27.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:27.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:27.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:27.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_15 2024-09-18T01:21:28.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 114 2024-09-18T01:21:28.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_15 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_15.status 2024-09-18T01:21:28.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:28.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:28.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:28.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_15.status 2024-09-18T01:21:28.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_15 --format xml 2024-09-18T01:21:28.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:28.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:21:28.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=114 2024-09-18T01:21:28.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:28.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:28.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:21:28.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.status 2024-09-18T01:21:28.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:28.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:28.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:28.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.status 2024-09-18T01:21:28.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2024-09-18T01:21:28.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:29.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:29.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:29.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:29.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:29.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:29.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:29.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 114 = '' 2024-09-18T01:21:29.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:29.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:21:29.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.status 2024-09-18T01:21:29.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:29.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:29.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:29.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.status 2024-09-18T01:21:29.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2024-09-18T01:21:29.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:29.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:29.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:29.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:29.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:29.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:29.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:29.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 114 = '' 2024-09-18T01:21:29.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:29.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:21:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.status 2024-09-18T01:21:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.status 2024-09-18T01:21:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2024-09-18T01:21:30.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:30.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=114 2024-09-18T01:21:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 114 = 114 2024-09-18T01:21:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap 2024-09-18T01:21:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:21:30.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:31.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap 2024-09-18T01:21:31.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_15': 2024-09-18T01:21:31.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:21:31.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 11c3b2ba069 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.11c3b2ba069 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:11:57 2024 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:11:57 2024 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:11:57 2024 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:21:31.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: b04593f0-b419-4bde-acad-9a17209e79b7 2024-09-18T01:21:31.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:21:31.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:31.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_15 snap 2024-09-18T01:21:31.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:31.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:21:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:21:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_15.export 2024-09-18T01:21:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.export 2024-09-18T01:21:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_15.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.export 2024-09-18T01:21:31.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_15.export 2024-09-18T01:21:31.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:31.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.export 2024-09-18T01:21:31.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:31.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_15.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.export 2024-09-18T01:21:31.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_15.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_15.export 2024-09-18T01:21:31.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:31.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:21:31.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_16 2024-09-18T01:21:31.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_16 snap 2024-09-18T01:21:31.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:31.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:31.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:31.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:31.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:21:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_16 2024-09-18T01:21:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started 2024-09-18T01:21:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:21:33.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:33.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:34.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_16 started 2024-09-18T01:21:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:21:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:21:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:21:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16 2024-09-18T01:21:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:21:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:21:34.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:34.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:21:34.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:21:34.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:21:34.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:34.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:21:34.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:21:34.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:21:34.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:34.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:34.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:34.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:34.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:34.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:34.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:21:34.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:21:34.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:21:34.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:34.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:34.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:34.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2024-09-18T01:21:34.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:21:34.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:21:34.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:21:34.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:21:34.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:21:34.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:21:34.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_16 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_16 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:34.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:34.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:34.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_16.status 2024-09-18T01:21:34.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_16.status 2024-09-18T01:21:34.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:34.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_16.status 2024-09-18T01:21:34.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_16.status 2024-09-18T01:21:34.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_16.status 2024-09-18T01:21:34.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:34.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.status 2024-09-18T01:21:34.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_16 2024-09-18T01:21:34.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:34.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:34.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:34.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_16 2024-09-18T01:21:35.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 116 2024-09-18T01:21:35.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_16 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_16.status 2024-09-18T01:21:35.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:35.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:35.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:35.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_16.status 2024-09-18T01:21:35.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_16 --format xml 2024-09-18T01:21:35.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:35.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:21:35.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=116 2024-09-18T01:21:35.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:35.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:35.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:21:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.status 2024-09-18T01:21:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.status 2024-09-18T01:21:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2024-09-18T01:21:36.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:36.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:36.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=50 2024-09-18T01:21:36.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 116 = 50 2024-09-18T01:21:36.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:36.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:21:36.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.status 2024-09-18T01:21:36.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:36.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.status 2024-09-18T01:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2024-09-18T01:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:36.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=50 2024-09-18T01:21:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 116 = 50 2024-09-18T01:21:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:36.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:21:37.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.status 2024-09-18T01:21:37.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:37.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:37.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:37.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.status 2024-09-18T01:21:37.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2024-09-18T01:21:37.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:37.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:37.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:37.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:37.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:37.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:37.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:37.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 116 = '' 2024-09-18T01:21:37.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:37.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:21:38.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.status 2024-09-18T01:21:38.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:38.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:38.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:38.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.status 2024-09-18T01:21:38.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2024-09-18T01:21:38.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:39.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:39.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=116 2024-09-18T01:21:39.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 116 = 116 2024-09-18T01:21:39.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:39.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap 2024-09-18T01:21:39.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:39.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:39.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:39.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:39.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:39.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:21:39.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap 2024-09-18T01:21:40.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_16': 2024-09-18T01:21:40.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1268cb5d6f62 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1268cb5d6f62 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:12:12 2024 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:12:12 2024 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:12:12 2024 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:21:40.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:21:40.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:21:40.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 1c96c904-8436-480e-934e-936299cf345d 2024-09-18T01:21:40.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:21:40.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:40.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_16 snap 2024-09-18T01:21:40.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:40.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:21:40.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:40.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:21:40.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_16.export 2024-09-18T01:21:40.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.export 2024-09-18T01:21:40.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_16.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.export 2024-09-18T01:21:40.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_16.export 2024-09-18T01:21:40.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:40.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.export 2024-09-18T01:21:40.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:40.458 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_16.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.export 2024-09-18T01:21:40.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_16.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_16.export 2024-09-18T01:21:40.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:40.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:21:40.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_17 2024-09-18T01:21:40.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_17 snap 2024-09-18T01:21:40.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:40.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:40.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:40.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:40.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:21:41.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_17 2024-09-18T01:21:41.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:41.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:41.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:41.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started 2024-09-18T01:21:41.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:41.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:41.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:41.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:42.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_17 started 2024-09-18T01:21:42.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:42.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:42.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:42.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:21:42.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:21:42.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:21:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17 2024-09-18T01:21:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:21:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:21:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:21:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:21:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:21:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:21:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:21:42.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:21:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17 2024-09-18T01:21:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:21:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:21:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:21:42.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:42.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:21:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:21:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:21:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:21:42.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:21:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:42.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17 2024-09-18T01:21:42.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-09-18T01:21:42.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:21:42.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:21:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_17 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:21:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:21:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:21:42.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:21:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:42.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:42.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_17 2024-09-18T01:21:42.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:21:42.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:21:42.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:21:42.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:21:42.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:21:42.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_17 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_17 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_17.status 2024-09-18T01:21:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_17.status 2024-09-18T01:21:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_17.status 2024-09-18T01:21:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_17.status 2024-09-18T01:21:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_17.status 2024-09-18T01:21:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.status 2024-09-18T01:21:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_17 2024-09-18T01:21:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:42.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:42.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:42.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_17 2024-09-18T01:21:43.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 118 2024-09-18T01:21:43.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_17 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_17.status 2024-09-18T01:21:43.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:43.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:43.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:43.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_17.status 2024-09-18T01:21:43.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_17 --format xml 2024-09-18T01:21:43.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:43.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:21:43.815 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=118 2024-09-18T01:21:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:43.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:21:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.status 2024-09-18T01:21:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.status 2024-09-18T01:21:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2024-09-18T01:21:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:44.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:44.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=51 2024-09-18T01:21:44.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 118 = 51 2024-09-18T01:21:44.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:44.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:21:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.status 2024-09-18T01:21:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.status 2024-09-18T01:21:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2024-09-18T01:21:44.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:44.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:44.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=51 2024-09-18T01:21:44.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 118 = 51 2024-09-18T01:21:44.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:44.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:21:45.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.status 2024-09-18T01:21:45.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:45.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:45.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:45.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.status 2024-09-18T01:21:45.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2024-09-18T01:21:45.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:45.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:45.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:45.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:45.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:45.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:45.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:45.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 118 = '' 2024-09-18T01:21:45.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:45.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:21:46.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.status 2024-09-18T01:21:46.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:46.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:46.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:46.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.status 2024-09-18T01:21:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2024-09-18T01:21:46.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:47.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=118 2024-09-18T01:21:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 118 = 118 2024-09-18T01:21:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap 2024-09-18T01:21:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:47.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:47.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:47.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:21:47.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:48.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap 2024-09-18T01:21:48.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_17': 2024-09-18T01:21:48.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:21:48.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:21:48.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:21:48.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc1d7c5255 2024-09-18T01:21:48.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc1d7c5255 2024-09-18T01:21:48.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:21:48.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:21:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:21:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:21:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:12:25 2024 2024-09-18T01:21:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:12:25 2024 2024-09-18T01:21:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:12:25 2024 2024-09-18T01:21:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:21:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:21:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:21:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 3e9f111f-659a-4c25-a089-ca421c2ebd1a 2024-09-18T01:21:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:21:48.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:48.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_17 snap 2024-09-18T01:21:48.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:48.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:21:48.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:48.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:21:48.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_17.export 2024-09-18T01:21:48.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.export 2024-09-18T01:21:48.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_17.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.export 2024-09-18T01:21:48.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_17.export 2024-09-18T01:21:48.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:48.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.export 2024-09-18T01:21:48.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:48.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_17.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.export 2024-09-18T01:21:48.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_17.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_17.export 2024-09-18T01:21:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:21:48.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_18 2024-09-18T01:21:48.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_18 snap 2024-09-18T01:21:48.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:48.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:48.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:48.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:48.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:21:49.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_18 2024-09-18T01:21:49.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:49.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:49.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started 2024-09-18T01:21:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:21:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:49.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:50.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_18 started 2024-09-18T01:21:50.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:50.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:50.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:50.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:21:50.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:21:50.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:21:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18 2024-09-18T01:21:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:21:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:21:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:21:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:21:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:21:50.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:50.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:21:50.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:21:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:21:50.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:21:50.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:50.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18 2024-09-18T01:21:50.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-18T01:21:50.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:21:50.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:21:50.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:50.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_18 2024-09-18T01:21:50.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:21:50.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:50.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:50.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:21:50.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:21:50.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:50.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:50.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:50.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:21:50.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:21:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:21:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:21:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:21:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:21:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:50.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:21:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:21:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:21:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:21:50.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:21:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:21:50.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:21:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:50.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_18 2024-09-18T01:21:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-09-18T01:21:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:21:50.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:21:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_18 2024-09-18T01:21:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:21:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:21:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:21:50.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:50.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:50.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:50.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:21:50.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:21:50.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:50.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:50.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:21:50.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:21:50.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:21:50.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:21:50.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:21:51.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:21:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:21:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:21:51.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:21:51.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_18 2024-09-18T01:21:51.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:21:51.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:21:51.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:21:51.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:21:51.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:21:51.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_18 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_18 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:51.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:51.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:51.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_18.status 2024-09-18T01:21:51.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_18.status 2024-09-18T01:21:51.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:51.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_18.status 2024-09-18T01:21:51.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_18.status 2024-09-18T01:21:51.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_18.status 2024-09-18T01:21:51.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:21:51.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.status 2024-09-18T01:21:51.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_18 2024-09-18T01:21:51.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:51.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:51.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:51.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_18 2024-09-18T01:21:51.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 120 2024-09-18T01:21:51.783 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_18 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_18.status 2024-09-18T01:21:51.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:51.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:51.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:51.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_18.status 2024-09-18T01:21:51.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_18 --format xml 2024-09-18T01:21:51.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:52.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:21:52.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=120 2024-09-18T01:21:52.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:52.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:52.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:21:52.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.status 2024-09-18T01:21:52.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:52.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:52.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:52.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.status 2024-09-18T01:21:52.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2024-09-18T01:21:52.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:52.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=52 2024-09-18T01:21:52.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 120 = 52 2024-09-18T01:21:52.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:52.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:21:52.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.status 2024-09-18T01:21:52.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:52.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:52.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:52.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.status 2024-09-18T01:21:52.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2024-09-18T01:21:52.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:52.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:52.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:52.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:52.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:52.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:52.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:52.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 120 = '' 2024-09-18T01:21:52.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:52.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:21:53.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.status 2024-09-18T01:21:53.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:53.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:53.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:53.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.status 2024-09-18T01:21:53.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2024-09-18T01:21:53.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:21:53.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:21:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:21:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:21:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:21:53.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 120 = '' 2024-09-18T01:21:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:21:53.593 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:21:55.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.status 2024-09-18T01:21:55.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:55.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:55.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:55.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.status 2024-09-18T01:21:55.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2024-09-18T01:21:55.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:21:55.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:21:55.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=120 2024-09-18T01:21:55.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 120 = 120 2024-09-18T01:21:55.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:55.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap 2024-09-18T01:21:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:21:55.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:56.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap 2024-09-18T01:21:56.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_18': 2024-09-18T01:21:56.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1140f9694bf9 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1140f9694bf9 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:12:33 2024 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:12:33 2024 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:12:33 2024 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:21:56.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 716a32d0-d4de-4f8b-8c5b-1431e7a8c859 2024-09-18T01:21:56.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:21:56.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:56.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_18 snap 2024-09-18T01:21:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:21:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:21:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_18.export 2024-09-18T01:21:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.export 2024-09-18T01:21:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_18.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.export 2024-09-18T01:21:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_18.export 2024-09-18T01:21:56.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:56.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.export 2024-09-18T01:21:56.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:21:56.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_18.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.export 2024-09-18T01:21:56.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_18.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_18.export 2024-09-18T01:21:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:21:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:21:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_19 2024-09-18T01:21:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_19 snap 2024-09-18T01:21:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:21:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:21:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:21:56.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:21:58.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_19 2024-09-18T01:21:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:21:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started 2024-09-18T01:21:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:21:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:21:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:21:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:58.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:21:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_19 started 2024-09-18T01:21:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:21:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:21:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:21:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:21:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:21:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:21:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19 2024-09-18T01:21:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:21:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:21:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:21:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:21:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:21:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:21:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:21:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:21:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:21:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19 2024-09-18T01:21:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:21:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:21:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:21:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:21:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:21:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:21:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:21:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:21:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:21:59.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:21:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:21:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:59.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:21:59.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:21:59.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:21:59.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:59.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:59.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:21:59.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19 2024-09-18T01:21:59.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2024-09-18T01:21:59.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:21:59.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:21:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:21:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_19 2024-09-18T01:21:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:21:59.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:21:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:21:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:21:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:21:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:21:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:21:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:21:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:21:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:21:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:21:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:59.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:21:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:21:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:21:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:21:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:21:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:21:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:21:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:21:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:21:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:21:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:21:59.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:21:59.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:21:59.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:21:59.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:21:59.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:21:59.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:21:59.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:21:59.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:59.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:59.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:21:59.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_19 2024-09-18T01:22:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:22:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:22:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:22:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:22:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:22:00.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:22:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:22:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_19 2024-09-18T01:22:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:22:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:22:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:22:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_19 2024-09-18T01:22:00.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:00.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:00.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:00.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:22:00.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_19.status 2024-09-18T01:22:00.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_19.status 2024-09-18T01:22:00.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:00.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_19.status 2024-09-18T01:22:00.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_19.status 2024-09-18T01:22:00.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_19.status 2024-09-18T01:22:00.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:00.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.status 2024-09-18T01:22:00.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_19 2024-09-18T01:22:00.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:00.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:00.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:22:00.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_19 2024-09-18T01:22:00.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 122 2024-09-18T01:22:00.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_19 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_19.status 2024-09-18T01:22:00.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:00.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:00.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:22:00.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_19.status 2024-09-18T01:22:00.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_19 --format xml 2024-09-18T01:22:00.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:00.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:22:00.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=122 2024-09-18T01:22:00.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:00.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:00.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:22:01.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.status 2024-09-18T01:22:01.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:01.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:01.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:22:01.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.status 2024-09-18T01:22:01.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2024-09-18T01:22:01.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:01.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:01.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=53 2024-09-18T01:22:01.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 122 = 53 2024-09-18T01:22:01.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:01.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:22:01.455 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8299475735226399) > weight (0.25) 2024-09-18T01:22:01.455 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.47068041345178635) > weight (0.25) 2024-09-18T01:22:01.456 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.4018885521501059) > weight (0.25) 2024-09-18T01:22:01.456 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.8966998131385624) > weight (0.25) 2024-09-18T01:22:01.456 INFO:tasks.rbd_mirror_thrash:waiting for 101 secs before thrashing 2024-09-18T01:22:01.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.status 2024-09-18T01:22:01.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:01.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:01.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:22:01.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.status 2024-09-18T01:22:01.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2024-09-18T01:22:01.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:01.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:01.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=53 2024-09-18T01:22:01.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 122 = 53 2024-09-18T01:22:01.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:01.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:22:02.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.status 2024-09-18T01:22:02.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:02.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:02.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:22:02.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.status 2024-09-18T01:22:02.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2024-09-18T01:22:02.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:02.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:02.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:02.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:22:02.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:22:02.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:22:02.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:22:02.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 122 = '' 2024-09-18T01:22:02.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:02.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:22:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.status 2024-09-18T01:22:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:22:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.status 2024-09-18T01:22:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2024-09-18T01:22:04.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:04.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=122 2024-09-18T01:22:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 122 = 122 2024-09-18T01:22:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:04.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap 2024-09-18T01:22:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:22:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:22:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:05.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap 2024-09-18T01:22:05.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_19': 2024-09-18T01:22:05.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:22:05.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:22:05.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:22:05.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 11c3278ef952 2024-09-18T01:22:05.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.11c3278ef952 2024-09-18T01:22:05.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:22:05.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:22:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:22:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:22:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:12:47 2024 2024-09-18T01:22:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:12:47 2024 2024-09-18T01:22:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:12:47 2024 2024-09-18T01:22:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:22:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:22:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:22:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 957171f2-b93c-457d-89be-5d45fdaad96c 2024-09-18T01:22:05.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:22:05.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:05.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_19 snap 2024-09-18T01:22:05.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:05.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:22:05.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:05.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:22:05.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_19.export 2024-09-18T01:22:05.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.export 2024-09-18T01:22:05.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_19.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.export 2024-09-18T01:22:05.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_19.export 2024-09-18T01:22:05.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:05.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.export 2024-09-18T01:22:05.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:05.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_19.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.export 2024-09-18T01:22:05.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_19.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_19.export 2024-09-18T01:22:05.638 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:05.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:22:05.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_20 2024-09-18T01:22:05.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_20 snap 2024-09-18T01:22:05.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:05.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:05.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:05.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:05.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_20 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_20 started 2024-09-18T01:22:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:22:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:22:07.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:22:07.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20 2024-09-18T01:22:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:22:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:22:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:22:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:22:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:22:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:22:07.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:22:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:22:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20 2024-09-18T01:22:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:22:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:22:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:22:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:07.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:22:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:22:07.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:07.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:07.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:07.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:07.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:07.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:07.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:07.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:22:07.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:22:07.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:22:07.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:07.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:07.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:07.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20 2024-09-18T01:22:07.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2024-09-18T01:22:07.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:22:07.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:22:07.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_20 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:07.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:07.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:07.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:07.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:22:07.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:22:07.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:22:07.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:07.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:07.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:07.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_20 2024-09-18T01:22:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:22:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:22:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:22:08.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:22:08.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:22:08.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:22:08.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_20 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_20 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:08.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_20.status 2024-09-18T01:22:08.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_20.status 2024-09-18T01:22:08.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:08.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_20.status 2024-09-18T01:22:08.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_20.status 2024-09-18T01:22:08.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_20.status 2024-09-18T01:22:08.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:08.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.status 2024-09-18T01:22:08.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_20 2024-09-18T01:22:08.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:08.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:08.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:08.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_20 2024-09-18T01:22:08.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 124 2024-09-18T01:22:08.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_20 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_20.status 2024-09-18T01:22:08.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:08.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:08.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:08.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_20.status 2024-09-18T01:22:08.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_20 --format xml 2024-09-18T01:22:08.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:08.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:22:08.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=124 2024-09-18T01:22:08.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:08.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:08.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:22:09.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.status 2024-09-18T01:22:09.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:09.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:09.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:09.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.status 2024-09-18T01:22:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2024-09-18T01:22:09.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:09.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:09.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=54 2024-09-18T01:22:09.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 124 = 54 2024-09-18T01:22:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:09.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:22:09.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.status 2024-09-18T01:22:09.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:09.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:09.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:09.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.status 2024-09-18T01:22:09.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2024-09-18T01:22:09.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:09.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:09.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=54 2024-09-18T01:22:09.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 124 = 54 2024-09-18T01:22:09.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:09.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:22:10.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.status 2024-09-18T01:22:10.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:10.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:10.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:10.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.status 2024-09-18T01:22:10.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2024-09-18T01:22:10.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:10.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:22:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:22:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:22:10.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:22:10.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 124 = '' 2024-09-18T01:22:10.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:10.437 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:22:12.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.status 2024-09-18T01:22:12.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:12.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:12.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:12.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.status 2024-09-18T01:22:12.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2024-09-18T01:22:12.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:12.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=124 2024-09-18T01:22:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 124 = 124 2024-09-18T01:22:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap 2024-09-18T01:22:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:22:12.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:13.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap 2024-09-18T01:22:13.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_20': 2024-09-18T01:22:13.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:22:13.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1268401299c4 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1268401299c4 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:13:02 2024 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:13:02 2024 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:13:02 2024 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:22:13.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 820484e1-ad03-4296-9676-f12697dabdb5 2024-09-18T01:22:13.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:22:13.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:13.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_20 snap 2024-09-18T01:22:13.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:13.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:22:13.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:13.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:22:13.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_20.export 2024-09-18T01:22:13.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.export 2024-09-18T01:22:13.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_20.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.export 2024-09-18T01:22:13.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_20.export 2024-09-18T01:22:13.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:13.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.export 2024-09-18T01:22:13.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:13.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_20.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.export 2024-09-18T01:22:13.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_20.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_20.export 2024-09-18T01:22:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:22:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_21 2024-09-18T01:22:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_21 snap 2024-09-18T01:22:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:13.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:22:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_21 2024-09-18T01:22:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started 2024-09-18T01:22:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:22:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:15.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_21 started 2024-09-18T01:22:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:22:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:22:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:22:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21 2024-09-18T01:22:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:22:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:22:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:22:15.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:22:15.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:22:15.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:15.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:22:15.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:22:15.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:22:15.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:22:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:22:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:22:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21 2024-09-18T01:22:15.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-09-18T01:22:15.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:22:15.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:22:15.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:15.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_21 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:15.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:15.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:15.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:22:15.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:22:15.962 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:22:15.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:15.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:15.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:15.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_21 2024-09-18T01:22:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:22:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:22:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:22:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:22:16.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:22:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:22:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:22:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_21 2024-09-18T01:22:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:22:16.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_21 2024-09-18T01:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_21.status 2024-09-18T01:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_21.status 2024-09-18T01:22:16.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_21.status 2024-09-18T01:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_21.status 2024-09-18T01:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_21.status 2024-09-18T01:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.status 2024-09-18T01:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_21 2024-09-18T01:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_21 2024-09-18T01:22:16.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 126 2024-09-18T01:22:16.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_21 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_21.status 2024-09-18T01:22:16.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:16.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:16.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:16.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_21.status 2024-09-18T01:22:16.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_21 --format xml 2024-09-18T01:22:16.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:16.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:22:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=126 2024-09-18T01:22:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:22:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.status 2024-09-18T01:22:17.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.status 2024-09-18T01:22:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2024-09-18T01:22:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:17.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:17.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=55 2024-09-18T01:22:17.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 126 = 55 2024-09-18T01:22:17.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:17.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:22:17.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.status 2024-09-18T01:22:17.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:17.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:17.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:17.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.status 2024-09-18T01:22:17.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2024-09-18T01:22:17.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:17.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:17.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=55 2024-09-18T01:22:17.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 126 = 55 2024-09-18T01:22:17.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:17.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:22:18.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.status 2024-09-18T01:22:18.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:18.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:18.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:18.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.status 2024-09-18T01:22:18.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2024-09-18T01:22:18.409 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:18.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:18.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:22:18.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:22:18.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:22:18.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:22:18.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 126 = '' 2024-09-18T01:22:18.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:18.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:22:20.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.status 2024-09-18T01:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.status 2024-09-18T01:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2024-09-18T01:22:20.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:20.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=126 2024-09-18T01:22:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 126 = 126 2024-09-18T01:22:20.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:20.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap 2024-09-18T01:22:20.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:20.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:20.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:20.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:20.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:20.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:22:20.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:21.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap 2024-09-18T01:22:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_21': 2024-09-18T01:22:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:22:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:22:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:22:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bcd05fb8cd 2024-09-18T01:22:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bcd05fb8cd 2024-09-18T01:22:21.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:22:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:22:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:22:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:22:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:13:11 2024 2024-09-18T01:22:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:13:11 2024 2024-09-18T01:22:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:13:11 2024 2024-09-18T01:22:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:22:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:22:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:22:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 56bbe368-35dd-4230-b5bc-91e7ad099a39 2024-09-18T01:22:21.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:22:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_21 snap 2024-09-18T01:22:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:22:21.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:21.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:22:21.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_21.export 2024-09-18T01:22:21.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.export 2024-09-18T01:22:21.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_21.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.export 2024-09-18T01:22:21.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_21.export 2024-09-18T01:22:21.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:21.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.export 2024-09-18T01:22:21.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_21.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.export 2024-09-18T01:22:21.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_21.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_21.export 2024-09-18T01:22:21.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:21.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:22:21.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_22 2024-09-18T01:22:21.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_22 snap 2024-09-18T01:22:21.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:21.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:21.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:21.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:21.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:22:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_22 2024-09-18T01:22:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:22.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started 2024-09-18T01:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:22.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:23.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_22 started 2024-09-18T01:22:23.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:23.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:23.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:23.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:22:23.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:22:23.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:22:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22 2024-09-18T01:22:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:22:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:22:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:22:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:22:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:22:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:22:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:22:23.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:22:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:23.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:23.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:23.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:22:23.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:23.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:22:23.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:22:23.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:23.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:23.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:22:23.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:22:23.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:23.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:23.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:23.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:23.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:23.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:23.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:23.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:22:23.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:22:23.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:22:23.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:23.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:23.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:23.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22 2024-09-18T01:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2024-09-18T01:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:22:23.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:22:23.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:23.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_22 2024-09-18T01:22:23.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:22:23.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:23.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:23.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:22:23.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:23.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:22:23.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:22:24.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:24.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:24.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:24.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_22 2024-09-18T01:22:24.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:22:24.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:22:24.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:22:24.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:22:24.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_22 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_22 2024-09-18T01:22:24.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_22.status 2024-09-18T01:22:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_22.status 2024-09-18T01:22:24.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_22.status 2024-09-18T01:22:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_22.status 2024-09-18T01:22:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_22.status 2024-09-18T01:22:24.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:24.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.status 2024-09-18T01:22:24.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_22 2024-09-18T01:22:24.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:24.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:24.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:24.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_22 2024-09-18T01:22:24.622 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.26101548764832494) > weight (0.25) 2024-09-18T01:22:24.622 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8554506130910894) > weight (0.25) 2024-09-18T01:22:24.623 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.2935918403651038) > weight (0.25) 2024-09-18T01:22:24.623 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.502928024606395) > weight (0.25) 2024-09-18T01:22:24.623 INFO:tasks.rbd_mirror_thrash:waiting for 83 secs before thrashing 2024-09-18T01:22:24.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 128 2024-09-18T01:22:24.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_22 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_22.status 2024-09-18T01:22:24.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:24.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:24.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:24.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_22.status 2024-09-18T01:22:24.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_22 --format xml 2024-09-18T01:22:24.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:24.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:22:24.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=128 2024-09-18T01:22:24.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:24.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:24.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:22:25.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.status 2024-09-18T01:22:25.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:25.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:25.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:25.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.status 2024-09-18T01:22:25.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2024-09-18T01:22:25.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:25.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:25.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=56 2024-09-18T01:22:25.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 128 = 56 2024-09-18T01:22:25.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:25.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:22:25.587 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.status 2024-09-18T01:22:25.587 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:25.587 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:25.587 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:25.587 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.status 2024-09-18T01:22:25.587 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2024-09-18T01:22:25.587 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:25.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=56 2024-09-18T01:22:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 128 = 56 2024-09-18T01:22:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:25.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:22:26.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.status 2024-09-18T01:22:26.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:26.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:26.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:26.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.status 2024-09-18T01:22:26.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2024-09-18T01:22:26.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:26.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:26.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:22:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:22:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:22:26.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:22:26.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 128 = '' 2024-09-18T01:22:26.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:26.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:22:28.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.status 2024-09-18T01:22:28.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:28.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:28.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:28.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.status 2024-09-18T01:22:28.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2024-09-18T01:22:28.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:28.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:28.151 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=128 2024-09-18T01:22:28.151 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 128 = 128 2024-09-18T01:22:28.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:28.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap 2024-09-18T01:22:28.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:28.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:28.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:28.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:28.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:28.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:22:28.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:29.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap 2024-09-18T01:22:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_22': 2024-09-18T01:22:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:22:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:22:29.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1140af577cd3 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1140af577cd3 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:13:26 2024 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:13:26 2024 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:13:26 2024 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:22:29.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 2b2d5ff2-a08d-4bda-9b74-e9d26e07305c 2024-09-18T01:22:29.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:22:29.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:29.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_22 snap 2024-09-18T01:22:29.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:29.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:22:29.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:29.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:22:29.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_22.export 2024-09-18T01:22:29.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.export 2024-09-18T01:22:29.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_22.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.export 2024-09-18T01:22:29.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_22.export 2024-09-18T01:22:29.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:29.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.export 2024-09-18T01:22:29.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:29.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_22.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.export 2024-09-18T01:22:29.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_22.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_22.export 2024-09-18T01:22:29.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:22:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_23 2024-09-18T01:22:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_23 snap 2024-09-18T01:22:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:22:30.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_23 2024-09-18T01:22:30.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:30.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:30.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:30.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started 2024-09-18T01:22:30.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:30.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:30.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:30.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:22:30.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:30.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:30.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:31.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_23 started 2024-09-18T01:22:31.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:31.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:31.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:31.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:22:31.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:22:31.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:22:31.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23 2024-09-18T01:22:31.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:22:31.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:31.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:22:31.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:22:31.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:22:31.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:22:31.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:31.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:22:31.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:22:31.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:31.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:31.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:31.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:22:31.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:31.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:22:31.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:22:31.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:31.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:31.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:22:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:22:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2024-09-18T01:22:32.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2024-09-18T01:22:32.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:22:32.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:22:32.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:32.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2024-09-18T01:22:32.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:22:32.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:22:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:22:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:22:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:22:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:22:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:32.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:32.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:32.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:32.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:22:32.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:22:32.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:22:32.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:32.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:32.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:32.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2024-09-18T01:22:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:22:32.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:22:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:22:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:22:32.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:22:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:22:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:22:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_23 2024-09-18T01:22:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:22:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:22:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_23 2024-09-18T01:22:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_23.status 2024-09-18T01:22:32.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_23.status 2024-09-18T01:22:32.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_23.status 2024-09-18T01:22:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_23.status 2024-09-18T01:22:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_23.status 2024-09-18T01:22:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.status 2024-09-18T01:22:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_23 2024-09-18T01:22:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_23 2024-09-18T01:22:32.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 130 2024-09-18T01:22:32.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_23 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_23.status 2024-09-18T01:22:32.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:32.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:32.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:32.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_23.status 2024-09-18T01:22:32.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_23 --format xml 2024-09-18T01:22:32.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:32.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:22:32.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=130 2024-09-18T01:22:32.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:32.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:32.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:22:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.status 2024-09-18T01:22:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.status 2024-09-18T01:22:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2024-09-18T01:22:33.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:33.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:33.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=57 2024-09-18T01:22:33.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 130 = 57 2024-09-18T01:22:33.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:33.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:22:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.status 2024-09-18T01:22:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:33.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.status 2024-09-18T01:22:33.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2024-09-18T01:22:33.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:33.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:33.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=57 2024-09-18T01:22:33.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 130 = 57 2024-09-18T01:22:33.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:33.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:22:33.743 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:22:33.741+0000 7f80d5d16640 -1 reset not still connected to 0x5641a011bee0 2024-09-18T01:22:33.744 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:22:33.741+0000 7f80d5d16640 -1 reset not still connected to 0x5641a297e000 2024-09-18T01:22:34.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.status 2024-09-18T01:22:34.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:34.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:34.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:34.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.status 2024-09-18T01:22:34.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2024-09-18T01:22:34.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:34.514 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:34.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:34.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:22:34.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:22:34.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:22:34.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:22:34.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 130 = '' 2024-09-18T01:22:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:22:36.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.status 2024-09-18T01:22:36.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:36.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:36.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:36.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.status 2024-09-18T01:22:36.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2024-09-18T01:22:36.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:36.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:36.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=130 2024-09-18T01:22:36.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 130 = 130 2024-09-18T01:22:36.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:36.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap 2024-09-18T01:22:36.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:36.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:36.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:36.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:36.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:36.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:22:36.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:37.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap 2024-09-18T01:22:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_23': 2024-09-18T01:22:37.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 11c35edf7536 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.11c35edf7536 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:13:39 2024 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:13:39 2024 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:13:39 2024 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:22:37.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:22:37.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: bf72281c-0895-415a-a775-377773847714 2024-09-18T01:22:37.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:22:37.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_23 snap 2024-09-18T01:22:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:22:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:22:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_23.export 2024-09-18T01:22:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.export 2024-09-18T01:22:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_23.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.export 2024-09-18T01:22:37.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_23.export 2024-09-18T01:22:37.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:37.404 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.export 2024-09-18T01:22:37.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:37.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_23.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.export 2024-09-18T01:22:37.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_23.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_23.export 2024-09-18T01:22:37.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:37.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:22:37.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_24 2024-09-18T01:22:37.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_24 snap 2024-09-18T01:22:37.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:37.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:37.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:37.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:37.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:22:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_24 2024-09-18T01:22:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started 2024-09-18T01:22:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:38.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:22:38.908 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:38.908 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:38.908 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:39.908 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_24 started 2024-09-18T01:22:39.908 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:39.908 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:39.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:39.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:22:39.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:22:39.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:22:39.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24 2024-09-18T01:22:39.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:22:39.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:39.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:22:39.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:22:39.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:22:39.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:22:39.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:39.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:22:39.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:22:39.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:22:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24 2024-09-18T01:22:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:22:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:22:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:22:39.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:22:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:39.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:22:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:22:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:22:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:22:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:22:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:39.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:39.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:39.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:39.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:39.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:22:39.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:22:39.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:22:39.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:39.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:39.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:39.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24 2024-09-18T01:22:40.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-09-18T01:22:40.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:22:40.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:22:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_24 2024-09-18T01:22:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:22:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:22:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:22:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:40.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:40.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:22:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:22:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:22:40.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:40.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:40.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:40.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_24 2024-09-18T01:22:40.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:22:40.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:22:40.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:22:40.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:22:40.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_24 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_24 2024-09-18T01:22:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_24.status 2024-09-18T01:22:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_24.status 2024-09-18T01:22:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:40.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_24.status 2024-09-18T01:22:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_24.status 2024-09-18T01:22:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_24.status 2024-09-18T01:22:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.status 2024-09-18T01:22:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_24 2024-09-18T01:22:40.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:40.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:40.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:40.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_24 2024-09-18T01:22:40.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 132 2024-09-18T01:22:40.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_24 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_24.status 2024-09-18T01:22:40.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:40.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:40.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:40.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_24.status 2024-09-18T01:22:40.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_24 --format xml 2024-09-18T01:22:40.925 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:40.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:22:40.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=132 2024-09-18T01:22:40.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:40.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:40.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:22:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.status 2024-09-18T01:22:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.status 2024-09-18T01:22:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2024-09-18T01:22:41.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:41.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:41.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=58 2024-09-18T01:22:41.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 132 = 58 2024-09-18T01:22:41.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:41.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:22:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.status 2024-09-18T01:22:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.status 2024-09-18T01:22:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2024-09-18T01:22:41.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:41.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:41.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=58 2024-09-18T01:22:41.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 132 = 58 2024-09-18T01:22:41.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:41.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:22:42.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.status 2024-09-18T01:22:42.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:42.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:42.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:42.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.status 2024-09-18T01:22:42.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2024-09-18T01:22:42.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:42.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:42.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:42.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:22:42.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:22:42.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:22:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:22:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 132 = '' 2024-09-18T01:22:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:42.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:22:42.777 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:22:42.775+0000 7f80d5d16640 -1 reset not still connected to 0x5641a2999ba0 2024-09-18T01:22:42.777 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:22:42.776+0000 7f80d5d16640 -1 reset not still connected to 0x5641ab140c30 2024-09-18T01:22:44.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.status 2024-09-18T01:22:44.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:44.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:44.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:44.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.status 2024-09-18T01:22:44.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2024-09-18T01:22:44.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:44.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:44.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=132 2024-09-18T01:22:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 132 = 132 2024-09-18T01:22:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap 2024-09-18T01:22:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:22:44.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:45.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap 2024-09-18T01:22:45.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_24': 2024-09-18T01:22:45.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1268d724267 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1268d724267 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:13:53 2024 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:13:53 2024 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:13:53 2024 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:22:45.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:22:45.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:22:45.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: e2f372b0-91b5-4e89-95d5-57ab346b50a4 2024-09-18T01:22:45.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:22:45.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:45.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_24 snap 2024-09-18T01:22:45.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:45.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:22:45.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:45.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:22:45.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_24.export 2024-09-18T01:22:45.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.export 2024-09-18T01:22:45.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_24.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.export 2024-09-18T01:22:45.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_24.export 2024-09-18T01:22:45.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:45.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.export 2024-09-18T01:22:45.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:45.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_24.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.export 2024-09-18T01:22:45.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_24.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_24.export 2024-09-18T01:22:45.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:45.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:22:45.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_25 2024-09-18T01:22:45.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_25 snap 2024-09-18T01:22:45.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:45.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:45.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:45.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:45.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:22:46.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_25 2024-09-18T01:22:46.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:46.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started 2024-09-18T01:22:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:22:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:46.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:47.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_25 started 2024-09-18T01:22:47.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:47.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:47.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:47.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:22:47.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:22:47.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:22:47.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25 2024-09-18T01:22:47.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:22:47.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:22:47.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:47.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:22:47.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:22:47.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:22:47.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:47.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:22:47.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:22:47.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:22:47.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:47.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25 2024-09-18T01:22:47.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:22:47.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:47.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:22:47.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:22:47.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:47.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:47.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:22:47.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:22:47.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:47.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:47.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:47.936 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:22:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:22:47.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:22:47.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:47.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:47.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:47.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25 2024-09-18T01:22:48.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2024-09-18T01:22:48.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:22:48.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:22:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_25 2024-09-18T01:22:48.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:22:48.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:48.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:22:48.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:22:48.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:22:48.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:48.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:48.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:48.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_25 2024-09-18T01:22:48.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2024-09-18T01:22:48.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:22:48.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_25 2024-09-18T01:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:22:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:22:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:22:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:22:48.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:22:48.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:22:48.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:22:48.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:22:48.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_25 2024-09-18T01:22:48.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:22:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:22:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:22:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:22:48.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:22:48.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:22:48.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:22:48.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:48.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_25 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_25 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_25.status 2024-09-18T01:22:48.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_25.status 2024-09-18T01:22:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_25.status 2024-09-18T01:22:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_25.status 2024-09-18T01:22:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_25.status 2024-09-18T01:22:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.status 2024-09-18T01:22:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_25 2024-09-18T01:22:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:48.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_25 2024-09-18T01:22:48.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 134 2024-09-18T01:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_25 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_25.status 2024-09-18T01:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_25.status 2024-09-18T01:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_25 --format xml 2024-09-18T01:22:48.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:49.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:22:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=134 2024-09-18T01:22:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:22:49.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.status 2024-09-18T01:22:49.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:49.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:49.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:49.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.status 2024-09-18T01:22:49.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2024-09-18T01:22:49.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:49.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=59 2024-09-18T01:22:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 134 = 59 2024-09-18T01:22:49.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:49.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:22:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.status 2024-09-18T01:22:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.status 2024-09-18T01:22:49.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2024-09-18T01:22:49.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:49.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:49.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=59 2024-09-18T01:22:49.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 134 = 59 2024-09-18T01:22:49.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:49.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:22:50.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.status 2024-09-18T01:22:50.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:50.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:50.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:50.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.status 2024-09-18T01:22:50.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2024-09-18T01:22:50.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:50.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:50.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:22:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:22:50.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:22:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:22:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 134 = '' 2024-09-18T01:22:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:50.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:22:52.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.status 2024-09-18T01:22:52.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:52.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:52.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:52.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.status 2024-09-18T01:22:52.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2024-09-18T01:22:52.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:52.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=134 2024-09-18T01:22:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 134 = 134 2024-09-18T01:22:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:52.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap 2024-09-18T01:22:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:22:52.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:53.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap 2024-09-18T01:22:53.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_25': 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc46be230e 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc46be230e 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:06 2024 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:06 2024 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:06 2024 2024-09-18T01:22:53.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:22:53.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:22:53.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:22:53.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 87947b3d-24d4-4600-a788-3246abc4083b 2024-09-18T01:22:53.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:22:53.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:53.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_25 snap 2024-09-18T01:22:53.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:53.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:22:53.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:53.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:22:53.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_25.export 2024-09-18T01:22:53.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.export 2024-09-18T01:22:53.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_25.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.export 2024-09-18T01:22:53.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_25.export 2024-09-18T01:22:53.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:53.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.export 2024-09-18T01:22:53.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:22:53.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_25.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.export 2024-09-18T01:22:53.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_25.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_25.export 2024-09-18T01:22:53.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:53.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:22:53.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_26 2024-09-18T01:22:53.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_26 snap 2024-09-18T01:22:53.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:53.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:53.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:22:53.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:22:53.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:22:54.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_26 2024-09-18T01:22:54.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:54.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:54.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:22:54.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started 2024-09-18T01:22:54.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:54.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:54.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:22:54.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:22:54.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:22:54.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:54.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:22:55.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_26 started 2024-09-18T01:22:55.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:55.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:55.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:22:55.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:22:55.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:22:55.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:22:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26 2024-09-18T01:22:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:22:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:22:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:22:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:22:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:22:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:22:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:22:55.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:22:55.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:55.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:55.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:55.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:22:55.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:22:55.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:22:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:22:55.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:22:55.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:55.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:55.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:22:55.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26 2024-09-18T01:22:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2024-09-18T01:22:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:22:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:22:56.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:56.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_26 2024-09-18T01:22:56.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:22:56.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:56.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:56.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:22:56.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:22:56.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:22:56.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:56.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:56.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:22:56.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_26 2024-09-18T01:22:56.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2024-09-18T01:22:56.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:22:56.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_26 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:22:56.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:22:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:22:56.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:22:56.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:22:56.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:22:56.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:22:56.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:22:56.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:22:56.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:22:56.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:22:56.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:22:56.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:22:56.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_26 2024-09-18T01:22:56.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:22:56.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:22:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:22:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:22:56.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:22:56.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:22:56.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:22:56.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:22:56.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_26 2024-09-18T01:22:56.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_26 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_26.status 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_26.status 2024-09-18T01:22:56.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:56.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_26.status 2024-09-18T01:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_26.status 2024-09-18T01:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_26.status 2024-09-18T01:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.status 2024-09-18T01:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_26 2024-09-18T01:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:22:56.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_26 2024-09-18T01:22:56.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 136 2024-09-18T01:22:56.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_26 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_26.status 2024-09-18T01:22:56.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:22:56.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:56.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:22:56.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_26.status 2024-09-18T01:22:56.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_26 --format xml 2024-09-18T01:22:56.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:57.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:22:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=136 2024-09-18T01:22:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:57.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:22:57.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.status 2024-09-18T01:22:57.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:57.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:57.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:22:57.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.status 2024-09-18T01:22:57.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2024-09-18T01:22:57.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:57.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=60 2024-09-18T01:22:57.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 136 = 60 2024-09-18T01:22:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:22:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.status 2024-09-18T01:22:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:22:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.status 2024-09-18T01:22:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2024-09-18T01:22:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:57.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:57.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=60 2024-09-18T01:22:57.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 136 = 60 2024-09-18T01:22:57.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:57.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:22:58.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.status 2024-09-18T01:22:58.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:22:58.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:22:58.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:22:58.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.status 2024-09-18T01:22:58.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2024-09-18T01:22:58.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:22:58.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:22:58.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:22:58.587 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:22:58.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:22:58.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:22:58.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:22:58.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 136 = '' 2024-09-18T01:22:58.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:22:58.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:23:00.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.status 2024-09-18T01:23:00.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:00.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:00.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:23:00.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.status 2024-09-18T01:23:00.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2024-09-18T01:23:00.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:00.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:00.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=136 2024-09-18T01:23:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 136 = 136 2024-09-18T01:23:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap 2024-09-18T01:23:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:23:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:23:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:01.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap 2024-09-18T01:23:01.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_26': 2024-09-18T01:23:01.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:23:01.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:23:01.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1283dee94888 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1283dee94888 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:13 2024 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:13 2024 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:13 2024 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:23:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:23:01.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: ca67ff60-0889-4b07-bb52-b0e5f8ac56c7 2024-09-18T01:23:01.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:23:01.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:01.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_26 snap 2024-09-18T01:23:01.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:01.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:23:01.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:01.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:23:01.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_26.export 2024-09-18T01:23:01.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.export 2024-09-18T01:23:01.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_26.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.export 2024-09-18T01:23:01.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_26.export 2024-09-18T01:23:01.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:01.479 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.export 2024-09-18T01:23:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:01.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_26.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.export 2024-09-18T01:23:01.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_26.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_26.export 2024-09-18T01:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_27 2024-09-18T01:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_27 snap 2024-09-18T01:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:23:02.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_27 2024-09-18T01:23:02.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:02.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:02.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:02.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started 2024-09-18T01:23:02.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:02.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:02.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:02.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:23:02.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:02.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:02.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:03.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_27 started 2024-09-18T01:23:03.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:03.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:03.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:23:03.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:23:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:23:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27 2024-09-18T01:23:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:23:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:23:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:23:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:03.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:03.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:23:03.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:03.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:03.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:03.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:23:03.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:03.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:03.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:23:03.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:23:03.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:23:04.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:04.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:04.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:04.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27 2024-09-18T01:23:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-18T01:23:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:04.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:04.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:04.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_27 2024-09-18T01:23:04.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:04.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:23:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:23:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:23:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:23:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:23:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:04.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:04.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:04.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:04.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:04.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:04.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:04.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:23:04.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:23:04.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:23:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:04.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_27 2024-09-18T01:23:04.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-18T01:23:04.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:04.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:04.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:04.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_27 2024-09-18T01:23:04.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:23:04.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:04.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:04.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:23:04.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:23:04.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:04.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:23:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:23:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:23:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:23:04.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:23:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:04.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_27 2024-09-18T01:23:04.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-09-18T01:23:04.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:04.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:04.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:04.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_27 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:04.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:04.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:23:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:23:04.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:23:04.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:04.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:04.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:04.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_27 2024-09-18T01:23:04.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:23:04.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:23:04.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:23:04.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:23:04.423 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:23:04.424 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:23:04.424 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:23:04.424 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:04.424 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_27 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_27 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_27.status 2024-09-18T01:23:04.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_27.status 2024-09-18T01:23:04.426 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:04.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_27.status 2024-09-18T01:23:04.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_27.status 2024-09-18T01:23:04.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_27.status 2024-09-18T01:23:04.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:04.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.status 2024-09-18T01:23:04.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_27 2024-09-18T01:23:04.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:04.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:04.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:04.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_27 2024-09-18T01:23:04.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 138 2024-09-18T01:23:05.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_27 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_27.status 2024-09-18T01:23:05.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:05.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:05.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:05.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_27.status 2024-09-18T01:23:05.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_27 --format xml 2024-09-18T01:23:05.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:05.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:23:05.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=138 2024-09-18T01:23:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:23:05.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.status 2024-09-18T01:23:05.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:05.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:05.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:05.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.status 2024-09-18T01:23:05.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2024-09-18T01:23:05.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:05.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=61 2024-09-18T01:23:05.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 138 = 61 2024-09-18T01:23:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:05.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:23:05.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.status 2024-09-18T01:23:05.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:05.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:05.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:05.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.status 2024-09-18T01:23:05.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2024-09-18T01:23:05.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:05.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:05.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:05.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 138 = '' 2024-09-18T01:23:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:05.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:23:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.status 2024-09-18T01:23:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.status 2024-09-18T01:23:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2024-09-18T01:23:06.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:06.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:06.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:06.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:06.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:06.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:06.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:06.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 138 = '' 2024-09-18T01:23:06.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:06.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:23:08.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.status 2024-09-18T01:23:08.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:08.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:08.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:08.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.status 2024-09-18T01:23:08.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2024-09-18T01:23:08.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:08.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:08.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=138 2024-09-18T01:23:08.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 138 = 138 2024-09-18T01:23:08.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:08.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap 2024-09-18T01:23:08.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:08.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:08.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:08.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:08.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:08.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:23:08.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:09.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap 2024-09-18T01:23:09.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_27': 2024-09-18T01:23:09.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:23:09.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:23:09.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:23:09.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12831fc30bcb 2024-09-18T01:23:09.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12831fc30bcb 2024-09-18T01:23:09.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:23:09.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:23:09.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:23:09.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:23:09.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:24 2024 2024-09-18T01:23:09.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:24 2024 2024-09-18T01:23:09.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:24 2024 2024-09-18T01:23:09.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:23:09.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:23:09.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:23:09.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: d521f001-ba66-4693-9d40-d7a054a004d9 2024-09-18T01:23:09.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:23:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_27 snap 2024-09-18T01:23:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:23:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:23:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_27.export 2024-09-18T01:23:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.export 2024-09-18T01:23:09.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_27.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.export 2024-09-18T01:23:09.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_27.export 2024-09-18T01:23:09.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:09.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.export 2024-09-18T01:23:09.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_27.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.export 2024-09-18T01:23:09.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_27.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_27.export 2024-09-18T01:23:09.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:09.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:23:09.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_28 2024-09-18T01:23:09.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_28 snap 2024-09-18T01:23:09.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:09.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:09.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:09.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:09.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:23:11.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_28 2024-09-18T01:23:11.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:11.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:11.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:11.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started 2024-09-18T01:23:11.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:11.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:11.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:11.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:23:11.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:11.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:11.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:12.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_28 started 2024-09-18T01:23:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:23:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:23:12.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:23:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28 2024-09-18T01:23:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:23:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:23:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:23:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:23:12.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:23:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:23:12.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:23:12.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:23:12.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:12.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:23:12.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:12.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:23:12.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:23:12.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:23:12.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:12.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:12.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:12.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2024-09-18T01:23:12.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2024-09-18T01:23:12.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:12.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:12.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:12.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:12.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:12.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:12.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:23:12.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:23:12.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:23:12.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:12.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:12.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:12.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2024-09-18T01:23:12.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2024-09-18T01:23:12.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:12.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_28 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:23:12.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:12.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:12.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:12.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:12.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:23:12.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:23:12.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:23:12.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:12.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:12.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:12.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_28 2024-09-18T01:23:12.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2024-09-18T01:23:12.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:12.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:12.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:12.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_28 2024-09-18T01:23:12.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:23:12.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:12.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:12.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:23:12.339 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:23:12.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:23:12.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:12.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:12.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:12.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_28 2024-09-18T01:23:12.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:23:12.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:23:12.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:23:12.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:23:12.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:23:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:23:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:23:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_28 2024-09-18T01:23:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:23:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:23:12.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_28 2024-09-18T01:23:12.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:12.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:12.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:12.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_28.status 2024-09-18T01:23:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_28.status 2024-09-18T01:23:12.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:12.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_28.status 2024-09-18T01:23:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_28.status 2024-09-18T01:23:12.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_28.status 2024-09-18T01:23:12.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:12.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.status 2024-09-18T01:23:12.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_28 2024-09-18T01:23:12.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:12.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_28 2024-09-18T01:23:13.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 140 2024-09-18T01:23:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_28 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_28.status 2024-09-18T01:23:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:13.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:13.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_28.status 2024-09-18T01:23:13.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_28 --format xml 2024-09-18T01:23:13.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:13.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:23:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=140 2024-09-18T01:23:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:23:13.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.status 2024-09-18T01:23:13.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:13.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:13.272 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:13.272 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.status 2024-09-18T01:23:13.272 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2024-09-18T01:23:13.272 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:13.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:13.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=62 2024-09-18T01:23:13.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 140 = 62 2024-09-18T01:23:13.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:13.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:23:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.status 2024-09-18T01:23:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.status 2024-09-18T01:23:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2024-09-18T01:23:13.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:13.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:13.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 140 = '' 2024-09-18T01:23:13.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:13.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:23:14.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.status 2024-09-18T01:23:14.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:14.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:14.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:14.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.status 2024-09-18T01:23:14.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2024-09-18T01:23:14.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:14.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:14.844 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:14.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:14.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 140 = '' 2024-09-18T01:23:14.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:14.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:23:16.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.status 2024-09-18T01:23:16.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:16.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:16.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:16.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.status 2024-09-18T01:23:16.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2024-09-18T01:23:16.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:16.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:16.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=140 2024-09-18T01:23:16.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 140 = 140 2024-09-18T01:23:16.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:16.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap 2024-09-18T01:23:16.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:16.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:23:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:17.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap 2024-09-18T01:23:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_28': 2024-09-18T01:23:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:23:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:23:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:23:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1283c73b396b 2024-09-18T01:23:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1283c73b396b 2024-09-18T01:23:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:23:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:23:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:23:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:23:17.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:34 2024 2024-09-18T01:23:17.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:34 2024 2024-09-18T01:23:17.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:34 2024 2024-09-18T01:23:17.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:23:17.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:23:17.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:23:17.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 24274cb0-c250-4f16-a3b6-16755405ece7 2024-09-18T01:23:17.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:23:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_28 snap 2024-09-18T01:23:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:23:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:23:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_28.export 2024-09-18T01:23:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.export 2024-09-18T01:23:17.553 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_28.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.export 2024-09-18T01:23:17.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_28.export 2024-09-18T01:23:17.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:17.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.export 2024-09-18T01:23:17.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:17.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_28.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.export 2024-09-18T01:23:18.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_28.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_28.export 2024-09-18T01:23:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:23:18.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_29 2024-09-18T01:23:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_29 snap 2024-09-18T01:23:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:18.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:23:20.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_29 2024-09-18T01:23:20.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:20.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:20.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:20.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started 2024-09-18T01:23:20.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:20.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:20.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:20.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:23:20.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:20.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:20.038 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_29 started 2024-09-18T01:23:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:23:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:23:21.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:23:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29 2024-09-18T01:23:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:23:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:23:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:23:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:23:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:23:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:23:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:23:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:23:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29 2024-09-18T01:23:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:23:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:21.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:23:21.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:21.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:23:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:23:21.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:23:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:21.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29 2024-09-18T01:23:21.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2024-09-18T01:23:21.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:21.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:21.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:21.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_29 2024-09-18T01:23:21.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:23:21.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:21.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:21.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:23:21.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:23:21.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:21.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:21.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:21.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:21.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:21.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:21.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:21.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:21.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:21.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:23:21.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:23:21.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:23:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:21.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_29 2024-09-18T01:23:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2024-09-18T01:23:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:21.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:21.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:21.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_29 2024-09-18T01:23:21.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:21.262 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:23:21.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:23:21.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:23:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:21.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:21.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:21.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_29 2024-09-18T01:23:21.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:23:21.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:23:21.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:23:21.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:23:21.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:23:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:23:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:23:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_29 2024-09-18T01:23:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:23:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:23:21.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_29 2024-09-18T01:23:21.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:21.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:21.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:21.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:21.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_29.status 2024-09-18T01:23:21.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_29.status 2024-09-18T01:23:21.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:21.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_29.status 2024-09-18T01:23:21.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_29.status 2024-09-18T01:23:21.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_29.status 2024-09-18T01:23:21.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:21.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.status 2024-09-18T01:23:21.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_29 2024-09-18T01:23:21.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:21.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:21.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:21.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_29 2024-09-18T01:23:22.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 142 2024-09-18T01:23:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_29 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_29.status 2024-09-18T01:23:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_29.status 2024-09-18T01:23:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_29 --format xml 2024-09-18T01:23:22.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:22.095 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:23:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=142 2024-09-18T01:23:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:22.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:23:22.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.status 2024-09-18T01:23:22.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:22.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:22.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:22.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.status 2024-09-18T01:23:22.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2024-09-18T01:23:22.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:22.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:22.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=63 2024-09-18T01:23:22.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 142 = 63 2024-09-18T01:23:22.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:22.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:23:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.status 2024-09-18T01:23:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.status 2024-09-18T01:23:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2024-09-18T01:23:22.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:22.796 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:22.796 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:22.797 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:22.797 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:22.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:22.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:22.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 142 = '' 2024-09-18T01:23:22.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:22.798 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:23:23.599 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.status 2024-09-18T01:23:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.status 2024-09-18T01:23:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2024-09-18T01:23:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:23.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:23.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:23.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:23.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 142 = '' 2024-09-18T01:23:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:23.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:23:25.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.status 2024-09-18T01:23:25.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:25.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:25.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:25.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.status 2024-09-18T01:23:25.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2024-09-18T01:23:25.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:25.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:25.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=142 2024-09-18T01:23:25.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 142 = 142 2024-09-18T01:23:25.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:25.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap 2024-09-18T01:23:25.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:25.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:25.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:25.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:25.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:25.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:23:25.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:26.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap 2024-09-18T01:23:26.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_29': 2024-09-18T01:23:26.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:23:26.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:23:26.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:23:26.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc4c7a8c46 2024-09-18T01:23:26.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc4c7a8c46 2024-09-18T01:23:26.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:23:26.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:47 2024 2024-09-18T01:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:47 2024 2024-09-18T01:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:47 2024 2024-09-18T01:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: f1b25004-306f-4390-bf36-878a869a0106 2024-09-18T01:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_29 snap 2024-09-18T01:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_29.export 2024-09-18T01:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.export 2024-09-18T01:23:26.363 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_29.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.export 2024-09-18T01:23:26.363 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_29.export 2024-09-18T01:23:26.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:26.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.export 2024-09-18T01:23:26.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:26.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_29.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.export 2024-09-18T01:23:26.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_29.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_29.export 2024-09-18T01:23:26.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:26.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:23:26.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_30 2024-09-18T01:23:26.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_30 snap 2024-09-18T01:23:26.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:26.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:26.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:26.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:26.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:23:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_30 2024-09-18T01:23:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:28.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started 2024-09-18T01:23:28.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:28.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:28.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:28.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:23:28.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:28.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:28.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:29.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_30 started 2024-09-18T01:23:29.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:23:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:23:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:23:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30 2024-09-18T01:23:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:23:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:23:29.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:23:29.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:23:29.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:23:29.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:29.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:23:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:23:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:23:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2024-09-18T01:23:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:23:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:23:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:23:29.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:29.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:23:29.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:23:29.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:29.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:23:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:23:29.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:23:29.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:29.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:29.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:29.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2024-09-18T01:23:29.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-18T01:23:29.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:29.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2024-09-18T01:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:23:29.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:23:29.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:29.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:29.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:29.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:29.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:29.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:29.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:29.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:23:29.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:23:29.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:23:29.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:29.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:29.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:29.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2024-09-18T01:23:29.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:29.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:29.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:29.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:29.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:29.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:29.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:29.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:23:29.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:23:29.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:23:29.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:29.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:29.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:29.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2024-09-18T01:23:29.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-09-18T01:23:29.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:29.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2024-09-18T01:23:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:23:29.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:23:29.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:23:29.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:23:29.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:29.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:29.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:29.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2024-09-18T01:23:29.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:23:29.490 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:23:29.490 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:23:29.490 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:23:29.490 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:23:29.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:23:29.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_30 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_30 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:29.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:29.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_30.status 2024-09-18T01:23:29.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_30.status 2024-09-18T01:23:29.493 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_30.status 2024-09-18T01:23:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_30.status 2024-09-18T01:23:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_30.status 2024-09-18T01:23:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.status 2024-09-18T01:23:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_30 2024-09-18T01:23:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:29.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_30 2024-09-18T01:23:30.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 144 2024-09-18T01:23:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_30 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_30.status 2024-09-18T01:23:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_30.status 2024-09-18T01:23:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_30 --format xml 2024-09-18T01:23:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:30.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:23:30.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=144 2024-09-18T01:23:30.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:30.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:30.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:23:30.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.status 2024-09-18T01:23:30.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:30.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:30.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:30.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.status 2024-09-18T01:23:30.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2024-09-18T01:23:30.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:30.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:30.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=64 2024-09-18T01:23:30.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 144 = 64 2024-09-18T01:23:30.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:30.371 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:23:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.status 2024-09-18T01:23:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.status 2024-09-18T01:23:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2024-09-18T01:23:30.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:30.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:30.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:30.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:30.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:30.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:30.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:30.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 144 = '' 2024-09-18T01:23:30.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:30.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:23:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.status 2024-09-18T01:23:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.status 2024-09-18T01:23:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2024-09-18T01:23:31.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:31.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:31.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:31.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:31.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:31.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:31.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:31.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 144 = '' 2024-09-18T01:23:31.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:31.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:23:33.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.status 2024-09-18T01:23:33.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:33.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:33.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:33.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.status 2024-09-18T01:23:33.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2024-09-18T01:23:33.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:33.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=144 2024-09-18T01:23:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 144 = 144 2024-09-18T01:23:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap 2024-09-18T01:23:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:33.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:33.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:23:33.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:34.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap 2024-09-18T01:23:34.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_30': 2024-09-18T01:23:34.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 11c358829059 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.11c358829059 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:14:59 2024 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:14:59 2024 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:14:59 2024 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:23:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:23:34.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:23:34.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: d6c020f1-f8ac-4074-858c-a7dfb0ecf52e 2024-09-18T01:23:34.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:23:34.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:34.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_30 snap 2024-09-18T01:23:34.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:34.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:23:34.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:34.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:23:34.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_30.export 2024-09-18T01:23:34.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.export 2024-09-18T01:23:34.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_30.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.export 2024-09-18T01:23:34.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_30.export 2024-09-18T01:23:34.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:34.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.export 2024-09-18T01:23:34.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:34.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_30.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.export 2024-09-18T01:23:34.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_30.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_30.export 2024-09-18T01:23:34.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:34.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:23:34.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_31 2024-09-18T01:23:34.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_31 snap 2024-09-18T01:23:34.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:34.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:34.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:34.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:34.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:23:36.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_31 2024-09-18T01:23:36.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:36.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:36.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:36.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started 2024-09-18T01:23:36.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:36.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:36.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:36.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:23:36.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:36.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:36.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:37.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_31 started 2024-09-18T01:23:37.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:37.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:37.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:37.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:23:37.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:23:37.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:23:37.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31 2024-09-18T01:23:37.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:23:37.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:37.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:23:37.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:23:37.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:23:37.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:23:37.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:37.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:23:37.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:23:37.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:23:37.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:37.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31 2024-09-18T01:23:37.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:23:37.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:37.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:37.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:23:37.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:23:37.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:37.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:37.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:37.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:23:37.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:23:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:23:37.546 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:23:37.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:37.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:37.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:37.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31 2024-09-18T01:23:37.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2024-09-18T01:23:37.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:37.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_31 2024-09-18T01:23:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:23:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:23:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:23:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:37.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:23:37.652 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:23:37.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:37.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:37.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:37.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:37.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:37.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:37.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:37.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:23:37.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:23:37.653 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:23:37.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:37.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:37.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:37.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_31 2024-09-18T01:23:37.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2024-09-18T01:23:37.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:37.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:37.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:37.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_31 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:37.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:37.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:23:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:23:37.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:23:37.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:37.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:37.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:37.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_31 2024-09-18T01:23:37.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:23:37.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:23:37.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:23:37.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:23:37.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_31 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_31 2024-09-18T01:23:37.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:37.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:37.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:37.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:37.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_31.status 2024-09-18T01:23:37.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_31.status 2024-09-18T01:23:37.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:37.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_31.status 2024-09-18T01:23:37.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_31.status 2024-09-18T01:23:37.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_31.status 2024-09-18T01:23:37.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:37.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.status 2024-09-18T01:23:37.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_31 2024-09-18T01:23:37.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:37.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:37.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:37.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_31 2024-09-18T01:23:38.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 146 2024-09-18T01:23:38.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_31 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_31.status 2024-09-18T01:23:38.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:38.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:38.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:38.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_31.status 2024-09-18T01:23:38.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_31 --format xml 2024-09-18T01:23:38.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:38.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:23:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=146 2024-09-18T01:23:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:38.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:23:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.status 2024-09-18T01:23:38.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.status 2024-09-18T01:23:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2024-09-18T01:23:38.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:38.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:38.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:38.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:38.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:38.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:38.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:38.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 146 = '' 2024-09-18T01:23:38.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:38.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:23:39.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.status 2024-09-18T01:23:39.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:39.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:39.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:39.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.status 2024-09-18T01:23:39.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2024-09-18T01:23:39.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:39.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:39.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:39.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:39.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:39.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:39.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:39.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 146 = '' 2024-09-18T01:23:39.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:39.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:23:40.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.status 2024-09-18T01:23:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.status 2024-09-18T01:23:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2024-09-18T01:23:40.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:40.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=146 2024-09-18T01:23:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 146 = 146 2024-09-18T01:23:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap 2024-09-18T01:23:40.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:40.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:40.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:40.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:40.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:40.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:23:40.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:41.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap 2024-09-18T01:23:41.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_31': 2024-09-18T01:23:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:23:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:23:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:23:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 126867120f51 2024-09-18T01:23:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.126867120f51 2024-09-18T01:23:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:23:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:23:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:23:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:23:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:15:13 2024 2024-09-18T01:23:41.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:15:13 2024 2024-09-18T01:23:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:15:13 2024 2024-09-18T01:23:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:23:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:23:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:23:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 753ffe95-b234-4f46-a463-e642f6aa8751 2024-09-18T01:23:41.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_31 snap 2024-09-18T01:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_31.export 2024-09-18T01:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.export 2024-09-18T01:23:41.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_31.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.export 2024-09-18T01:23:41.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_31.export 2024-09-18T01:23:41.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:41.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.export 2024-09-18T01:23:41.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_31.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.export 2024-09-18T01:23:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_31.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_31.export 2024-09-18T01:23:41.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:41.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:23:41.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_32 2024-09-18T01:23:41.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_32 snap 2024-09-18T01:23:41.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:41.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:41.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:41.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:41.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:23:42.457 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6066676167111498) > weight (0.25) 2024-09-18T01:23:42.457 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-09-18T01:23:42.458 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-09-18T01:23:42.458 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-18T01:23:42.459 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:23:42.458+0000 7f2db4892640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 50121) UID: 1000 2024-09-18T01:23:43.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_32 2024-09-18T01:23:43.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:43.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:43.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:43.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started 2024-09-18T01:23:43.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:23:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:44.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_32 started 2024-09-18T01:23:44.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:23:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:23:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:23:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32 2024-09-18T01:23:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:23:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:23:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:23:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:23:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:23:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:23:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:23:44.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:23:44.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:44.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:44.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:23:44.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:44.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:23:44.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:23:44.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:44.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:44.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:44.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32 2024-09-18T01:23:44.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2024-09-18T01:23:44.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:44.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32 2024-09-18T01:23:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:23:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:23:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:23:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:44.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:23:44.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:23:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:23:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:23:44.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:44.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:44.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:44.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32 2024-09-18T01:23:44.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-09-18T01:23:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_32 2024-09-18T01:23:44.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:23:44.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:44.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:23:44.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:23:44.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:23:44.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:44.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:44.783 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:44.783 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_32 2024-09-18T01:23:44.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2024-09-18T01:23:44.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:44.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:44.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_32 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:44.870 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:44.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:44.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:44.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:44.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:23:44.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:23:44.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:23:44.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:44.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:44.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:44.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_32 2024-09-18T01:23:44.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:23:44.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:23:44.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:23:44.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:23:44.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:23:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:23:44.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_32 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_32 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:44.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:44.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_32.status 2024-09-18T01:23:44.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_32.status 2024-09-18T01:23:44.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_32.status 2024-09-18T01:23:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_32.status 2024-09-18T01:23:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_32.status 2024-09-18T01:23:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:23:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.status 2024-09-18T01:23:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_32 2024-09-18T01:23:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:44.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_32 2024-09-18T01:23:45.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 148 2024-09-18T01:23:45.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_32 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_32.status 2024-09-18T01:23:45.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:45.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:45.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:45.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_32.status 2024-09-18T01:23:45.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_32 --format xml 2024-09-18T01:23:45.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:45.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:23:45.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=148 2024-09-18T01:23:45.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:45.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:45.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:23:45.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.status 2024-09-18T01:23:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.status 2024-09-18T01:23:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2024-09-18T01:23:45.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:45.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:45.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:45.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:45.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:45.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:45.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:45.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 148 = '' 2024-09-18T01:23:45.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:45.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:23:46.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.status 2024-09-18T01:23:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.status 2024-09-18T01:23:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2024-09-18T01:23:46.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:46.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:23:46.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:46.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:23:46.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:23:46.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:23:46.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:23:46.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 148 = '' 2024-09-18T01:23:46.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:23:46.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:23:47.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.status 2024-09-18T01:23:47.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:47.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:47.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:47.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.status 2024-09-18T01:23:47.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2024-09-18T01:23:47.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:23:47.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:23:47.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=148 2024-09-18T01:23:47.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 148 = 148 2024-09-18T01:23:47.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:47.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap 2024-09-18T01:23:47.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:47.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:47.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:47.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:47.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:47.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:23:47.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:47.624 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8831222890993244) > weight (0.25) 2024-09-18T01:23:47.624 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6546106317641334) > weight (0.25) 2024-09-18T01:23:47.624 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5360722332860415) > weight (0.25) 2024-09-18T01:23:47.624 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.9790580351381625) > weight (0.25) 2024-09-18T01:23:47.624 INFO:tasks.rbd_mirror_thrash:waiting for 88 secs before thrashing 2024-09-18T01:23:48.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap 2024-09-18T01:23:48.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_32': 2024-09-18T01:23:48.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:23:48.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:23:48.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:23:48.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 128368ec95e 2024-09-18T01:23:48.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.128368ec95e 2024-09-18T01:23:48.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:23:48.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:23:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:23:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:23:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:15:27 2024 2024-09-18T01:23:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:15:27 2024 2024-09-18T01:23:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:15:27 2024 2024-09-18T01:23:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:23:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:23:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:23:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 884d933d-d88c-4657-97ca-5aa2e3991921 2024-09-18T01:23:48.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:23:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_32 snap 2024-09-18T01:23:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:23:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:23:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_32.export 2024-09-18T01:23:48.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.export 2024-09-18T01:23:48.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_32.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.export 2024-09-18T01:23:48.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_32.export 2024-09-18T01:23:48.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:48.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.export 2024-09-18T01:23:48.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:23:48.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_32.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.export 2024-09-18T01:23:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_32.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_32.export 2024-09-18T01:23:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:23:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:23:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_33 2024-09-18T01:23:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_33 snap 2024-09-18T01:23:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:23:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:23:48.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:23:48.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:23:50.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_33 2024-09-18T01:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started 2024-09-18T01:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:23:51.459 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-09-18T01:23:51.459 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:23:51.459 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-09-18T01:23:51.459 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-09-18T01:23:51.460 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-09-18T01:23:51.463 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-09-18T01:23:51.463 INFO:tasks.rbd_mirror_thrash:waiting for 110 secs before thrashing 2024-09-18T01:23:51.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_33 started 2024-09-18T01:23:51.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:23:51.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:23:51.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:23:51.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:23:51.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:23:51.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:23:51.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33 2024-09-18T01:23:51.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:23:51.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:51.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:23:51.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:23:51.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:23:51.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:23:51.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:51.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:23:51.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:23:51.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:23:51.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:51.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2024-09-18T01:23:51.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:23:51.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:51.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:51.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:23:51.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:23:51.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:51.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:51.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:51.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:23:51.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:51.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:51.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:51.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:23:51.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:51.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:23:51.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:23:51.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:51.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:51.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:23:51.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:23:51.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:51.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:51.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:51.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:51.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:51.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:51.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:51.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:23:51.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:23:51.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:23:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:51.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2024-09-18T01:23:51.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:51.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:51.689 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:51.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:51.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2024-09-18T01:23:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:23:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:23:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:23:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:23:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:51.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:51.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:51.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:51.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:51.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:51.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:23:51.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:23:51.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:23:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:51.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2024-09-18T01:23:51.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:51.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:51.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:51.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:51.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2024-09-18T01:23:51.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:23:51.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:51.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:51.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:23:51.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:23:51.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:51.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:51.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:51.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:23:51.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:51.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:51.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:51.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:23:51.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:51.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:51.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:23:51.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:23:51.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:23:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2024-09-18T01:23:51.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:51.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:51.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_33 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:23:51.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:51.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:51.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:51.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:51.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:51.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:51.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:23:51.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:23:51.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:23:51.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:51.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:51.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:51.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_33 2024-09-18T01:23:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:51.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:51.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 1 2024-09-18T01:23:52.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:23:52.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:23:53.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:53.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2024-09-18T01:23:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:23:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:23:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:23:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:23:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:53.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:23:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:23:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:23:53.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:53.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:53.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:53.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2024-09-18T01:23:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:53.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:53.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:53.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2024-09-18T01:23:53.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:23:53.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:53.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:53.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:23:53.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:23:53.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:23:53.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:23:53.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:53.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:53.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:53.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:53.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:53.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:53.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:53.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:23:53.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:23:53.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:23:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2024-09-18T01:23:53.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:53.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:53.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2024-09-18T01:23:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:23:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:23:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:23:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:53.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:23:53.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:23:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:23:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:23:53.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:53.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:53.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:53.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2024-09-18T01:23:53.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:53.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:53.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_33 2024-09-18T01:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:23:53.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:23:53.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:53.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:53.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:23:53.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:23:53.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:23:53.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:53.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:53.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:53.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_33 2024-09-18T01:23:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:53.410 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:53.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:53.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 2 2024-09-18T01:23:55.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:23:55.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:23:55.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:55.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2024-09-18T01:23:55.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:55.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:55.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:23:55.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:23:55.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:23:55.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:55.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:55.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:55.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2024-09-18T01:23:55.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:55.528 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:55.528 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:23:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:23:55.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:23:55.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:55.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:55.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:55.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2024-09-18T01:23:55.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:55.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:55.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:55.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:55.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2024-09-18T01:23:55.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:23:55.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:55.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:55.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:55.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:23:55.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:23:55.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:55.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:55.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:23:55.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2024-09-18T01:23:55.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:55.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:55.725 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:55.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:55.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_33 2024-09-18T01:23:55.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:23:55.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:55.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:55.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:23:55.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:23:55.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:55.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:23:55.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:23:55.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:55.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:55.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:55.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:23:55.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:23:55.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:55.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:55.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:23:55.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:23:55.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:23:55.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_33 2024-09-18T01:23:55.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:23:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 4 2024-09-18T01:23:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:23:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:23:59.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:59.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2024-09-18T01:23:59.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:23:59.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:59.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:59.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:23:59.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:23:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:23:59.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:59.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:59.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:23:59.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2024-09-18T01:23:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:23:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:23:59.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:23:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:23:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2024-09-18T01:23:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:23:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:23:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:23:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:23:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:23:59.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:23:59.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:23:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:23:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:23:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:23:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:23:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:23:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:23:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:23:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:23:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:23:59.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:23:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:23:59.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2024-09-18T01:24:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:24:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:00.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:24:00.046 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:00.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:24:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:24:00.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:24:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:00.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:00.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2024-09-18T01:24:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:24:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:00.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:00.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_33 2024-09-18T01:24:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:24:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:24:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:24:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:24:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:00.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:24:00.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:24:00.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:00.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:00.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:24:00.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:24:00.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:24:00.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:24:00.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:24:00.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:24:00.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_33 2024-09-18T01:24:00.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:24:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:00.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:00.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 8 2024-09-18T01:24:08.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:24:08.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:24:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2024-09-18T01:24:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:24:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:24:08.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:24:08.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:24:08.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:08.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:08.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:08.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2024-09-18T01:24:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:24:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:08.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:08.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:08.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2024-09-18T01:24:08.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:24:08.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:08.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:08.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:24:08.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:24:08.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:08.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:08.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:08.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:24:08.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:08.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:08.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:08.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:24:08.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:08.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:24:08.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:24:08.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:24:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:24:08.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:08.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:08.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:08.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2024-09-18T01:24:08.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:24:08.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:08.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:08.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:08.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2024-09-18T01:24:08.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:24:08.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:08.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:08.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:24:08.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:24:08.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:08.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:08.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:08.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:24:08.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:08.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:24:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:24:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:24:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:24:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:24:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:08.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:08.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:08.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:08.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:08.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:08.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:08.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:24:08.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:24:08.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:24:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:08.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2024-09-18T01:24:08.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:24:08.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:08.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:08.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_33 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:24:08.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:08.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:24:08.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:24:08.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:08.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:08.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:24:08.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:24:08.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:24:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:24:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:24:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:24:08.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_33 2024-09-18T01:24:08.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:24:08.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:08.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 8 2024-09-18T01:24:16.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:24:16.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:24:16.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:16.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2024-09-18T01:24:16.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:24:16.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:16.676 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:24:16.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:24:16.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:16.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:16.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:16.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2024-09-18T01:24:16.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-09-18T01:24:16.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:16.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:16.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:16.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2024-09-18T01:24:16.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:24:16.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:16.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:16.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:24:16.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:24:16.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:24:16.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:16.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:16.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:16.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2024-09-18T01:24:16.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:24:16.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:24:16.900 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:24:16.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:24:16.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:24:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:24:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:24:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_33 2024-09-18T01:24:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:24:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:24:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:24:16.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_33 2024-09-18T01:24:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:24:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_33.status 2024-09-18T01:24:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_33.status 2024-09-18T01:24:16.904 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:16.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_33.status 2024-09-18T01:24:16.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_33.status 2024-09-18T01:24:16.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_33.status 2024-09-18T01:24:16.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:16.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.status 2024-09-18T01:24:16.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_33 2024-09-18T01:24:16.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:16.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:16.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:24:16.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_33 2024-09-18T01:24:17.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 150 2024-09-18T01:24:17.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_33 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_33.status 2024-09-18T01:24:17.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:17.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:17.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:24:17.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_33.status 2024-09-18T01:24:17.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_33 --format xml 2024-09-18T01:24:17.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:17.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:24:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=150 2024-09-18T01:24:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:17.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:24:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.status 2024-09-18T01:24:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:24:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.status 2024-09-18T01:24:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2024-09-18T01:24:17.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:17.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:17.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:17.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:24:17.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:24:17.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:24:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:24:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 150 = '' 2024-09-18T01:24:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:17.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:24:18.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.status 2024-09-18T01:24:18.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:18.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:18.315 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:24:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.status 2024-09-18T01:24:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2024-09-18T01:24:18.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:18.363 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:18.363 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:18.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:24:18.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:24:18.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:24:18.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:24:18.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 150 = '' 2024-09-18T01:24:18.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:18.365 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:24:19.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.status 2024-09-18T01:24:19.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:19.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:19.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:24:19.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.status 2024-09-18T01:24:19.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2024-09-18T01:24:19.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:19.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:19.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=150 2024-09-18T01:24:19.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 150 = 150 2024-09-18T01:24:19.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap 2024-09-18T01:24:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:24:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:24:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:24:19.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:24:20.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap 2024-09-18T01:24:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_33': 2024-09-18T01:24:20.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc24208d4f 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc24208d4f 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:15:36 2024 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:15:36 2024 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:15:36 2024 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:24:20.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:24:20.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 4d961941-4020-4df3-821f-ef25e429389f 2024-09-18T01:24:20.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:24:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_33 snap 2024-09-18T01:24:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:24:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:24:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_33.export 2024-09-18T01:24:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.export 2024-09-18T01:24:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_33.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.export 2024-09-18T01:24:20.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_33.export 2024-09-18T01:24:20.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:24:20.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.export 2024-09-18T01:24:20.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:24:20.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_33.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.export 2024-09-18T01:24:20.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_33.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_33.export 2024-09-18T01:24:20.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:20.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:24:20.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_34 2024-09-18T01:24:20.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_34 snap 2024-09-18T01:24:20.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:20.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:20.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:20.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:20.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:24:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_34 2024-09-18T01:24:22.616 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_34 started 2024-09-18T01:24:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:24:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:24:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:22.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:24:23.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_34 started 2024-09-18T01:24:23.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:23.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:23.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:23.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:24:23.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:24:23.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:24:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_34 2024-09-18T01:24:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:24:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:24:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:24:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:24:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:24:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:24:23.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:24:23.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:24:23.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:23.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_34 2024-09-18T01:24:23.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:24:23.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:23.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:24:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:24:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:24:23.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:24:23.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:23.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:23.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:23.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_34 2024-09-18T01:24:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-18T01:24:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:23.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:23.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_34 2024-09-18T01:24:23.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:24:23.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:23.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:23.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:24:23.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:24:23.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:23.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:23.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:23.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:24:23.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:23.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:23.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:23.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:23.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:23.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:23.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:24:23.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:24:23.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:24:23.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:23.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:23.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:23.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_34 2024-09-18T01:24:23.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-18T01:24:23.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:23.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_34 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:24:23.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:23.842 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:23.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:23.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:23.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:23.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:23.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:24:23.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:24:23.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:24:23.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:23.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:23.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:23.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_34 2024-09-18T01:24:23.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-09-18T01:24:23.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:23.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_34 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:24:23.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:23.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:23.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:24:23.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:24:23.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:23.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:23.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:24:23.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:24:23.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:24:23.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:24:23.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:24:23.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:24:23.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_34 2024-09-18T01:24:24.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:24:24.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:24:24.055 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:24:24.055 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:24:24.055 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:24:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:24:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:24:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_34 2024-09-18T01:24:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_34 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_34.status 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_34.status 2024-09-18T01:24:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_34.status 2024-09-18T01:24:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_34.status 2024-09-18T01:24:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_34.status 2024-09-18T01:24:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.status 2024-09-18T01:24:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_34 2024-09-18T01:24:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:24.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_34 2024-09-18T01:24:24.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 152 2024-09-18T01:24:24.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_34 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_34.status 2024-09-18T01:24:24.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:24.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:24.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:24.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_34.status 2024-09-18T01:24:24.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_34 --format xml 2024-09-18T01:24:24.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:24.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:24:24.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=152 2024-09-18T01:24:24.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:24.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:24.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:24:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.status 2024-09-18T01:24:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:24.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.status 2024-09-18T01:24:24.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2024-09-18T01:24:24.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:24:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:24:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:24:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:24:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 152 = '' 2024-09-18T01:24:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:24:25.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.status 2024-09-18T01:24:25.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:25.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:25.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:25.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.status 2024-09-18T01:24:25.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2024-09-18T01:24:25.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:25.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:25.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:24:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:24:25.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:24:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:24:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 152 = '' 2024-09-18T01:24:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:25.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:24:26.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.status 2024-09-18T01:24:26.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:26.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:26.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:26.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.status 2024-09-18T01:24:26.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2024-09-18T01:24:26.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:26.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=152 2024-09-18T01:24:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 152 = 152 2024-09-18T01:24:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_34 snap 2024-09-18T01:24:26.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:24:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:24:26.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:24:27.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_34@snap 2024-09-18T01:24:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_34': 2024-09-18T01:24:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:24:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:24:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:24:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 11c36e8167a9 2024-09-18T01:24:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.11c36e8167a9 2024-09-18T01:24:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:24:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:24:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:24:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:24:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:15:42 2024 2024-09-18T01:24:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:15:42 2024 2024-09-18T01:24:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:15:42 2024 2024-09-18T01:24:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:24:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:24:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:24:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 4770a819-7931-426e-a8e9-0d6c6ec6d2de 2024-09-18T01:24:27.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:24:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_34 snap 2024-09-18T01:24:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:24:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:24:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_34.export 2024-09-18T01:24:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.export 2024-09-18T01:24:27.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_34.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.export 2024-09-18T01:24:27.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_34@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_34.export 2024-09-18T01:24:27.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:24:27.454 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_34@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.export 2024-09-18T01:24:27.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:24:27.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_34.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.export 2024-09-18T01:24:27.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_34.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_34.export 2024-09-18T01:24:27.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:27.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:24:27.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_35 2024-09-18T01:24:27.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_35 snap 2024-09-18T01:24:27.760 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:27.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_35@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:24:29.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_35 2024-09-18T01:24:29.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:29.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:29.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:29.639 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_35 started 2024-09-18T01:24:29.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:29.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:29.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:29.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:24:29.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:24:29.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:29.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:24:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_35 started 2024-09-18T01:24:30.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:24:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:24:30.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:24:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_35 2024-09-18T01:24:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:24:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:24:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:24:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:24:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:24:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:30.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:24:30.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:24:30.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:24:30.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:30.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_35 2024-09-18T01:24:30.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:24:30.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:24:30.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:24:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:24:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:24:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:24:30.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:24:30.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:30.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:30.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:30.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_35 2024-09-18T01:24:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-18T01:24:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_35 2024-09-18T01:24:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:24:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:24:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:24:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:30.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:24:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:24:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:30.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_35 2024-09-18T01:24:30.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-09-18T01:24:30.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:30.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:30.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:30.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_35 2024-09-18T01:24:30.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:24:30.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:30.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:30.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:24:30.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:24:30.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:30.863 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:24:30.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:24:30.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:30.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:30.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:30.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:30.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:30.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:30.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:30.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:24:30.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:24:30.864 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:24:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:30.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_35 2024-09-18T01:24:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:24:30.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:24:30.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:24:30.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:24:30.974 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_35 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_35 2024-09-18T01:24:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_35.status 2024-09-18T01:24:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_35.status 2024-09-18T01:24:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:30.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_35.status 2024-09-18T01:24:30.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_35.status 2024-09-18T01:24:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_35.status 2024-09-18T01:24:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.status 2024-09-18T01:24:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_35 2024-09-18T01:24:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_35 2024-09-18T01:24:31.971 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 154 2024-09-18T01:24:31.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_35 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_35.status 2024-09-18T01:24:31.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:31.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:31.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:31.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_35.status 2024-09-18T01:24:31.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_35 --format xml 2024-09-18T01:24:31.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:32.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:24:32.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=154 2024-09-18T01:24:32.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:32.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:32.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:24:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.status 2024-09-18T01:24:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:32.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.status 2024-09-18T01:24:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2024-09-18T01:24:32.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:32.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:32.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=69 2024-09-18T01:24:32.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 154 = 69 2024-09-18T01:24:32.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:32.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:24:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.status 2024-09-18T01:24:32.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:32.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:32.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:32.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.status 2024-09-18T01:24:32.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2024-09-18T01:24:32.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:32.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:32.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=69 2024-09-18T01:24:32.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 154 = 69 2024-09-18T01:24:32.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:32.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:24:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.status 2024-09-18T01:24:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.status 2024-09-18T01:24:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2024-09-18T01:24:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:33.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:33.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:33.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:24:33.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:24:33.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:24:33.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:24:33.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 154 = '' 2024-09-18T01:24:33.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:33.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:24:35.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.status 2024-09-18T01:24:35.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:35.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:35.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:35.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.status 2024-09-18T01:24:35.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2024-09-18T01:24:35.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:35.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=154 2024-09-18T01:24:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 154 = 154 2024-09-18T01:24:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_35 snap 2024-09-18T01:24:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:35.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:24:35.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:24:35.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:24:36.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_35@snap 2024-09-18T01:24:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_35': 2024-09-18T01:24:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1268ce9e5497 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1268ce9e5497 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:15:48 2024 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:15:48 2024 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:15:48 2024 2024-09-18T01:24:36.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:24:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:24:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:24:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 855097c3-5440-41cd-a767-29e218b7cf40 2024-09-18T01:24:36.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:24:36.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:36.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_35 snap 2024-09-18T01:24:36.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:36.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:24:36.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:36.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:24:36.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_35.export 2024-09-18T01:24:36.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.export 2024-09-18T01:24:36.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_35.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.export 2024-09-18T01:24:36.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_35@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_35.export 2024-09-18T01:24:36.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:24:36.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_35@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.export 2024-09-18T01:24:36.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:24:36.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_35.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.export 2024-09-18T01:24:36.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_35.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_35.export 2024-09-18T01:24:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:24:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_36 2024-09-18T01:24:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_36 snap 2024-09-18T01:24:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_36@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:24:37.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_36 2024-09-18T01:24:37.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:37.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:37.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:37.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_36 started 2024-09-18T01:24:37.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:37.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:37.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:37.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:24:37.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:24:37.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:37.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:24:38.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_36 started 2024-09-18T01:24:38.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:38.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:38.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:38.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:24:38.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:24:38.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:24:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_36 2024-09-18T01:24:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:24:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:24:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:24:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:24:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:24:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:24:38.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:24:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:24:38.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2024-09-18T01:24:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:24:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:24:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:24:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:24:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:38.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:38.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:38.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:24:38.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:38.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:24:38.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:24:38.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:38.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:38.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:24:38.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:24:38.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:24:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:24:39.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:24:39.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:39.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:39.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:39.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2024-09-18T01:24:39.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-09-18T01:24:39.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:39.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2024-09-18T01:24:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:24:39.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:39.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:39.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:24:39.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:24:39.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:39.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:24:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:24:39.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:39.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:24:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:24:39.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:24:39.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:39.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:39.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:39.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2024-09-18T01:24:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:24:39.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:24:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:24:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:24:39.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:24:39.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:24:39.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:24:39.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:39.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_36 2024-09-18T01:24:39.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:39.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:39.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:39.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:39.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:24:39.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:24:39.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_36 2024-09-18T01:24:39.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:39.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:39.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:39.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:39.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_36.status 2024-09-18T01:24:39.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_36.status 2024-09-18T01:24:39.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:39.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_36.status 2024-09-18T01:24:39.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_36.status 2024-09-18T01:24:39.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_36.status 2024-09-18T01:24:39.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:39.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.status 2024-09-18T01:24:39.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_36 2024-09-18T01:24:39.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:39.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:39.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:39.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_36 2024-09-18T01:24:39.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 156 2024-09-18T01:24:40.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_36 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_36.status 2024-09-18T01:24:40.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:40.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:40.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:40.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_36.status 2024-09-18T01:24:40.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_36 --format xml 2024-09-18T01:24:40.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:40.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:24:40.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=156 2024-09-18T01:24:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:40.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:24:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.status 2024-09-18T01:24:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.status 2024-09-18T01:24:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-09-18T01:24:40.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:40.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:40.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=70 2024-09-18T01:24:40.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 156 = 70 2024-09-18T01:24:40.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:40.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:24:40.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.status 2024-09-18T01:24:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.status 2024-09-18T01:24:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-09-18T01:24:40.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:40.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:40.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=70 2024-09-18T01:24:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 156 = 70 2024-09-18T01:24:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:40.754 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:24:41.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.status 2024-09-18T01:24:41.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:41.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:41.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:41.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.status 2024-09-18T01:24:41.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-09-18T01:24:41.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:41.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:41.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:24:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:24:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:24:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:24:41.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 156 = '' 2024-09-18T01:24:41.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:41.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:24:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.status 2024-09-18T01:24:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.status 2024-09-18T01:24:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-09-18T01:24:43.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:43.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:43.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=156 2024-09-18T01:24:43.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 156 = 156 2024-09-18T01:24:43.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:43.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_36 snap 2024-09-18T01:24:43.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:43.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:43.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:43.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:43.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:24:43.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:24:43.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:24:44.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_36@snap 2024-09-18T01:24:44.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_36': 2024-09-18T01:24:44.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:24:44.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:24:44.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:24:44.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 11c338f31ea8 2024-09-18T01:24:44.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.11c338f31ea8 2024-09-18T01:24:44.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:24:44.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:24:44.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:24:44.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:24:44.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:27 2024 2024-09-18T01:24:44.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:27 2024 2024-09-18T01:24:44.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:27 2024 2024-09-18T01:24:44.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:24:44.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:24:44.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:24:44.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 3ee14da1-57b2-4d60-bd27-8af0515acf51 2024-09-18T01:24:44.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:24:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_36 snap 2024-09-18T01:24:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:24:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:24:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_36.export 2024-09-18T01:24:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.export 2024-09-18T01:24:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_36.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.export 2024-09-18T01:24:44.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_36@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_36.export 2024-09-18T01:24:44.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:24:44.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_36@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.export 2024-09-18T01:24:44.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:24:44.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_36.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.export 2024-09-18T01:24:44.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_36.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_36.export 2024-09-18T01:24:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:24:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_37 2024-09-18T01:24:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_37 snap 2024-09-18T01:24:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:44.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:44.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_37@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:24:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_37 2024-09-18T01:24:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:46.003 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_37 started 2024-09-18T01:24:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:24:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:24:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:24:47.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_37 started 2024-09-18T01:24:47.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:47.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:47.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:47.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:24:47.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:24:47.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:24:47.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_37 2024-09-18T01:24:47.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:24:47.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:24:47.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:47.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:24:47.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:24:47.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:24:47.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:47.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:24:47.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:24:47.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:47.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:47.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:47.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:47.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:24:47.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:24:47.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:24:47.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:47.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:47.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:47.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2024-09-18T01:24:47.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2024-09-18T01:24:47.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:47.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:47.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:47.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_37 2024-09-18T01:24:47.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:24:47.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:47.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:24:47.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:24:47.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:47.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:47.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:47.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_37 2024-09-18T01:24:47.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2024-09-18T01:24:47.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:47.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_37 2024-09-18T01:24:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:24:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:24:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:24:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:47.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:24:47.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:47.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:47.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:47.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:24:47.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:47.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:24:47.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:24:47.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:47.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:47.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:24:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:24:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:24:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:24:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:24:47.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:47.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:47.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:47.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_37 2024-09-18T01:24:47.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:24:47.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:24:47.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:24:47.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:24:47.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:24:47.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:24:47.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:24:47.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:47.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_37 2024-09-18T01:24:47.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:47.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:47.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:47.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:47.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:24:47.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:24:47.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_37 2024-09-18T01:24:47.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:47.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:47.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:47.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:47.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_37.status 2024-09-18T01:24:47.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_37.status 2024-09-18T01:24:47.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:47.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_37.status 2024-09-18T01:24:47.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_37.status 2024-09-18T01:24:47.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_37.status 2024-09-18T01:24:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.status 2024-09-18T01:24:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_37 2024-09-18T01:24:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_37 2024-09-18T01:24:48.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 158 2024-09-18T01:24:48.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_37 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_37.status 2024-09-18T01:24:48.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:48.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:48.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:48.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_37.status 2024-09-18T01:24:48.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_37 --format xml 2024-09-18T01:24:48.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:48.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:24:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=158 2024-09-18T01:24:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:48.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:24:48.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.status 2024-09-18T01:24:48.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:48.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:48.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:48.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.status 2024-09-18T01:24:48.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-09-18T01:24:48.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:48.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:48.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=71 2024-09-18T01:24:48.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 158 = 71 2024-09-18T01:24:48.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:48.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:24:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.status 2024-09-18T01:24:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.status 2024-09-18T01:24:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-09-18T01:24:48.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:48.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:24:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:24:48.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:24:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:24:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 158 = '' 2024-09-18T01:24:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:48.988 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:24:49.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.status 2024-09-18T01:24:49.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:49.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:49.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:49.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.status 2024-09-18T01:24:49.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-09-18T01:24:49.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:49.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:49.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:49.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:24:49.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:24:49.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:24:49.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:24:49.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 158 = '' 2024-09-18T01:24:49.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:49.862 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:24:51.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.status 2024-09-18T01:24:51.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:51.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:51.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:51.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.status 2024-09-18T01:24:51.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-09-18T01:24:51.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:51.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:51.521 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=158 2024-09-18T01:24:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 158 = 158 2024-09-18T01:24:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_37 snap 2024-09-18T01:24:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:24:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:24:51.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:24:52.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_37@snap 2024-09-18T01:24:52.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_37': 2024-09-18T01:24:52.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:24:52.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:24:52.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:24:52.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 129317f6cff1 2024-09-18T01:24:52.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.129317f6cff1 2024-09-18T01:24:52.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:24:52.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:24:52.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:24:52.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:24:52.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:06 2024 2024-09-18T01:24:52.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:06 2024 2024-09-18T01:24:52.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:06 2024 2024-09-18T01:24:52.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:24:52.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:24:52.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:24:52.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 7af955aa-6145-46fc-be77-ed5d53ea99dd 2024-09-18T01:24:52.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:24:52.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:52.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_37 snap 2024-09-18T01:24:52.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:52.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:24:52.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:52.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:24:52.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_37.export 2024-09-18T01:24:52.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.export 2024-09-18T01:24:52.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_37.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.export 2024-09-18T01:24:52.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_37@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_37.export 2024-09-18T01:24:52.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:24:52.752 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_37@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.export 2024-09-18T01:24:52.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:24:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_37.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.export 2024-09-18T01:24:53.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_37.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_37.export 2024-09-18T01:24:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:24:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_38 2024-09-18T01:24:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_38 snap 2024-09-18T01:24:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:53.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:24:53.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:24:53.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_38@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:24:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_38 2024-09-18T01:24:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:24:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_38 started 2024-09-18T01:24:55.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:55.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:55.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:24:55.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:24:55.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:24:55.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:55.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:24:56.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_38 started 2024-09-18T01:24:56.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:24:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:24:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:24:56.021 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:24:56.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_38 2024-09-18T01:24:56.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:24:56.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:24:56.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:56.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:24:56.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:24:56.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:24:56.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:24:56.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:24:56.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:24:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:24:56.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_38 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:56.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:24:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:24:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:24:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:56.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:56.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:24:56.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:24:56.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:56.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:56.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:56.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:24:56.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:24:56.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:56.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:56.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:24:56.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:24:56.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:24:56.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:56.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:56.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:24:56.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_38 2024-09-18T01:24:56.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-18T01:24:56.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:56.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:56.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:56.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_38 2024-09-18T01:24:56.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:24:56.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:24:56.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:24:56.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:24:56.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:56.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:56.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:24:56.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_38 2024-09-18T01:24:56.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-09-18T01:24:56.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:24:56.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:24:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:24:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_38 2024-09-18T01:24:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:24:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:24:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:24:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:24:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:24:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:24:56.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:24:56.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:24:56.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:24:56.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:24:56.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:24:56.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:24:56.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:24:56.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:24:56.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:24:56.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:24:56.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:24:56.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:56.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:56.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:24:56.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_38 2024-09-18T01:24:56.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:24:56.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:24:56.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:24:56.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:24:56.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:24:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:24:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:24:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:24:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_38 2024-09-18T01:24:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:24:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:24:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:24:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_38 2024-09-18T01:24:56.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:24:56.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:56.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:56.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:24:56.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_38.status 2024-09-18T01:24:56.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_38.status 2024-09-18T01:24:56.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:56.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_38.status 2024-09-18T01:24:56.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_38.status 2024-09-18T01:24:56.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_38.status 2024-09-18T01:24:56.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:24:56.361 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.status 2024-09-18T01:24:56.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_38 2024-09-18T01:24:56.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:56.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:56.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:24:56.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_38 2024-09-18T01:24:57.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 160 2024-09-18T01:24:57.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_38 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_38.status 2024-09-18T01:24:57.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:24:57.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:57.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:24:57.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_38.status 2024-09-18T01:24:57.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_38 --format xml 2024-09-18T01:24:57.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:57.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:24:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=160 2024-09-18T01:24:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:57.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:24:57.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.status 2024-09-18T01:24:57.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:57.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:57.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:24:57.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.status 2024-09-18T01:24:57.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-09-18T01:24:57.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:57.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:57.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=72 2024-09-18T01:24:57.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 160 = 72 2024-09-18T01:24:57.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:57.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:24:57.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.status 2024-09-18T01:24:57.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:57.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:57.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:24:57.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.status 2024-09-18T01:24:57.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-09-18T01:24:57.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:57.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:57.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:24:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:24:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:24:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:24:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 160 = '' 2024-09-18T01:24:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:57.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:24:58.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.status 2024-09-18T01:24:58.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:24:58.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:24:58.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:24:58.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.status 2024-09-18T01:24:58.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-09-18T01:24:58.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:24:58.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:24:58.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:24:58.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:24:58.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:24:58.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:24:58.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:24:58.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 160 = '' 2024-09-18T01:24:58.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:24:58.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:25:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.status 2024-09-18T01:25:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:25:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.status 2024-09-18T01:25:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-09-18T01:25:00.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:00.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:00.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=160 2024-09-18T01:25:00.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 160 = 160 2024-09-18T01:25:00.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:00.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_38 snap 2024-09-18T01:25:00.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:00.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:00.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:25:00.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:25:00.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:01.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_38@snap 2024-09-18T01:25:01.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_38': 2024-09-18T01:25:01.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:25:01.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:25:01.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bcfbdf05fc 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bcfbdf05fc 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:21 2024 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:21 2024 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:21 2024 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:25:01.353 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 4ff798bb-162f-4ad6-acae-69c5a62867d9 2024-09-18T01:25:01.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:25:01.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:01.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_38 snap 2024-09-18T01:25:01.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:01.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:25:01.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:01.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:25:01.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_38.export 2024-09-18T01:25:01.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.export 2024-09-18T01:25:01.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_38.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.export 2024-09-18T01:25:01.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_38@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_38.export 2024-09-18T01:25:01.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:01.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_38@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.export 2024-09-18T01:25:01.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:01.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_38.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.export 2024-09-18T01:25:01.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_38.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_38.export 2024-09-18T01:25:01.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:01.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:25:01.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_39 2024-09-18T01:25:01.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_39 snap 2024-09-18T01:25:01.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:01.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:01.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:01.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:01.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_39@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:25:03.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_39 2024-09-18T01:25:03.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:03.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:03.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:03.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_39 started 2024-09-18T01:25:03.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:03.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:03.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:03.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:25:03.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:03.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:03.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:04.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_39 started 2024-09-18T01:25:04.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:04.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:04.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:04.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:25:04.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:25:04.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:25:04.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_39 2024-09-18T01:25:04.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:25:04.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:25:04.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:04.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:25:04.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:25:04.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:25:04.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:04.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:25:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:25:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:25:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2024-09-18T01:25:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:25:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:04.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:25:04.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:25:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:04.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2024-09-18T01:25:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-09-18T01:25:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:25:04.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:25:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2024-09-18T01:25:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:25:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:25:04.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:04.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:25:04.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:25:04.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:04.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2024-09-18T01:25:04.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:25:04.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:25:04.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:25:04.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:25:04.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:25:04.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:25:04.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:25:04.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:04.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_39 2024-09-18T01:25:04.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:04.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:04.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:04.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:04.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:25:04.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:25:04.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_39 2024-09-18T01:25:04.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:04.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:04.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:04.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:04.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_39.status 2024-09-18T01:25:04.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_39.status 2024-09-18T01:25:04.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:04.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_39.status 2024-09-18T01:25:04.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_39.status 2024-09-18T01:25:04.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_39.status 2024-09-18T01:25:04.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:04.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.status 2024-09-18T01:25:04.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_39 2024-09-18T01:25:04.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:04.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:04.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:04.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_39 2024-09-18T01:25:05.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 162 2024-09-18T01:25:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_39 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_39.status 2024-09-18T01:25:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:05.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:05.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_39.status 2024-09-18T01:25:05.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_39 --format xml 2024-09-18T01:25:05.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:25:05.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=162 2024-09-18T01:25:05.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:05.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:05.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:25:05.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.status 2024-09-18T01:25:05.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:05.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:05.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:05.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.status 2024-09-18T01:25:05.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-09-18T01:25:05.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:05.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=73 2024-09-18T01:25:05.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 162 = 73 2024-09-18T01:25:05.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:05.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:25:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.status 2024-09-18T01:25:05.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.status 2024-09-18T01:25:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-09-18T01:25:05.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:05.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=73 2024-09-18T01:25:05.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 162 = 73 2024-09-18T01:25:05.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:05.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:25:06.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.status 2024-09-18T01:25:06.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:06.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.status 2024-09-18T01:25:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-09-18T01:25:06.620 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:06.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:06.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:25:06.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:25:06.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:25:06.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:25:06.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 162 = '' 2024-09-18T01:25:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:06.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:25:08.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.status 2024-09-18T01:25:08.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:08.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:08.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:08.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.status 2024-09-18T01:25:08.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-09-18T01:25:08.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:08.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:08.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=162 2024-09-18T01:25:08.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 162 = 162 2024-09-18T01:25:08.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:08.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_39 snap 2024-09-18T01:25:08.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:08.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:08.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:08.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:08.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:08.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:25:08.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:09.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_39@snap 2024-09-18T01:25:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_39': 2024-09-18T01:25:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:25:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:25:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:25:09.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 126839e3f081 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.126839e3f081 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:28 2024 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:28 2024 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:28 2024 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 6f579a7e-81b1-4594-9e4f-741a03c86452 2024-09-18T01:25:09.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:25:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_39 snap 2024-09-18T01:25:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:25:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:25:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_39.export 2024-09-18T01:25:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.export 2024-09-18T01:25:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_39.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.export 2024-09-18T01:25:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_39@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_39.export 2024-09-18T01:25:09.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:09.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_39@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.export 2024-09-18T01:25:09.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:09.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_39.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.export 2024-09-18T01:25:09.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_39.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_39.export 2024-09-18T01:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_40 2024-09-18T01:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_40 snap 2024-09-18T01:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:09.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_40@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:25:11.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_40 2024-09-18T01:25:11.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:11.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_40 started 2024-09-18T01:25:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:25:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:11.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_40 started 2024-09-18T01:25:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:25:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:25:12.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:25:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_40 2024-09-18T01:25:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:25:12.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:25:12.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:12.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:25:12.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:25:12.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:25:12.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:12.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:25:12.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:12.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:25:12.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:12.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:25:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:25:12.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:25:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:12.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2024-09-18T01:25:12.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-18T01:25:12.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:25:12.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:25:12.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:12.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2024-09-18T01:25:12.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:25:12.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:25:12.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:12.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:25:12.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:25:12.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:25:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2024-09-18T01:25:12.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-18T01:25:12.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:25:12.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:25:12.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:12.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2024-09-18T01:25:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:25:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:25:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:25:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:25:12.289 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:12.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:25:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:25:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:25:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:25:12.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:25:12.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:25:12.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:25:12.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:25:12.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2024-09-18T01:25:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-09-18T01:25:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:25:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:25:12.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:12.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_40 2024-09-18T01:25:12.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:25:12.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:12.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:25:12.394 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:12.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:12.396 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:25:12.396 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:25:12.396 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:25:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:25:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:25:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:25:12.411 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_40 2024-09-18T01:25:12.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:25:12.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:25:12.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:25:12.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:25:12.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:25:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:25:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:25:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_40 2024-09-18T01:25:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:25:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:25:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_40 2024-09-18T01:25:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_40.status 2024-09-18T01:25:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_40.status 2024-09-18T01:25:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:12.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_40.status 2024-09-18T01:25:12.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_40.status 2024-09-18T01:25:12.506 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_40.status 2024-09-18T01:25:12.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:12.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.status 2024-09-18T01:25:12.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_40 2024-09-18T01:25:12.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:12.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:12.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:12.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_40 2024-09-18T01:25:13.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 164 2024-09-18T01:25:13.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_40 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_40.status 2024-09-18T01:25:13.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:13.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:13.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:13.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_40.status 2024-09-18T01:25:13.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_40 --format xml 2024-09-18T01:25:13.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:13.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:25:13.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=164 2024-09-18T01:25:13.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:13.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:13.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:25:13.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.status 2024-09-18T01:25:13.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:13.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:13.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:13.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.status 2024-09-18T01:25:13.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2024-09-18T01:25:13.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:13.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:13.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=74 2024-09-18T01:25:13.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 164 = 74 2024-09-18T01:25:13.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:13.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:25:13.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.status 2024-09-18T01:25:13.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:13.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:13.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:13.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.status 2024-09-18T01:25:13.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2024-09-18T01:25:13.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:13.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:13.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=74 2024-09-18T01:25:13.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 164 = 74 2024-09-18T01:25:13.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:13.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:25:14.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.status 2024-09-18T01:25:14.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:14.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:14.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:14.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.status 2024-09-18T01:25:14.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2024-09-18T01:25:14.632 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:14.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:14.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:14.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:25:14.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:25:14.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:25:14.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:25:14.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 164 = '' 2024-09-18T01:25:14.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:14.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1.6 2024-09-18T01:25:15.626 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6200265545187813) > weight (0.25) 2024-09-18T01:25:15.626 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6821751741385618) > weight (0.25) 2024-09-18T01:25:15.626 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6371743048735654) > weight (0.25) 2024-09-18T01:25:15.626 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2024-09-18T01:25:15.627 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2024-09-18T01:25:15.627 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-09-18T01:25:15.628 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:25:15.626+0000 7f9dfb8c8640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 47285) UID: 1000 2024-09-18T01:25:16.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.status 2024-09-18T01:25:16.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.status 2024-09-18T01:25:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2024-09-18T01:25:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:16.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:16.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=164 2024-09-18T01:25:16.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 164 = 164 2024-09-18T01:25:16.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:16.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_40 snap 2024-09-18T01:25:16.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:16.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:16.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:16.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:16.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:16.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:25:16.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:17.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_40@snap 2024-09-18T01:25:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_40': 2024-09-18T01:25:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:25:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:25:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:25:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1293db413083 2024-09-18T01:25:17.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1293db413083 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:41 2024 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:41 2024 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:41 2024 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 75569094-ed91-4db4-95b0-2512a6bdcc51 2024-09-18T01:25:17.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:25:17.384 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_40 snap 2024-09-18T01:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_40.export 2024-09-18T01:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.export 2024-09-18T01:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_40.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.export 2024-09-18T01:25:17.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_40@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_40.export 2024-09-18T01:25:17.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:17.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_40@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.export 2024-09-18T01:25:17.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:17.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_40.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.export 2024-09-18T01:25:17.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_40.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_40.export 2024-09-18T01:25:17.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:17.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:25:17.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_41 2024-09-18T01:25:17.871 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_41 snap 2024-09-18T01:25:17.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:17.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:17.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:17.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:17.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_41@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:25:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_41 2024-09-18T01:25:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_41 started 2024-09-18T01:25:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:25:19.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:19.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_41 started 2024-09-18T01:25:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:25:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:25:20.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:25:20.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_41 2024-09-18T01:25:20.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:25:20.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:20.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:25:20.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:25:20.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:25:20.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:25:20.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:20.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:25:20.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:25:20.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:25:20.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:20.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_41 2024-09-18T01:25:20.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:25:20.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:20.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:20.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:25:20.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:25:20.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:20.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:20.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:20.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:25:20.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:20.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:20.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:20.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:25:20.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:20.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:25:20.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:25:20.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:20.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:20.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:25:20.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:25:20.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:20.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:20.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:20.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:20.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:20.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:20.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:20.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:25:20.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:25:20.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:25:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:20.592 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_41 2024-09-18T01:25:20.628 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2024-09-18T01:25:20.628 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:25:20.628 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2024-09-18T01:25:20.628 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2024-09-18T01:25:20.629 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2024-09-18T01:25:20.631 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2024-09-18T01:25:20.631 INFO:tasks.rbd_mirror_thrash:waiting for 96 secs before thrashing 2024-09-18T01:25:20.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2024-09-18T01:25:20.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:25:20.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:25:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_41 2024-09-18T01:25:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:25:20.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:25:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:25:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:20.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:20.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:25:20.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:25:20.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:20.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:20.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:25:20.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:25:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:25:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:25:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:25:20.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:25:20.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:20.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:20.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:20.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:25:20.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:25:20.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:20.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:20.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:25:20.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:25:20.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:25:20.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:20.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:20.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:20.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_41 2024-09-18T01:25:20.813 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:25:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:25:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:25:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:25:20.814 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:25:20.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:25:20.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:25:20.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:20.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_41 2024-09-18T01:25:20.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:20.816 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_41 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_41.status 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_41.status 2024-09-18T01:25:20.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:20.818 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_41.status 2024-09-18T01:25:20.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_41.status 2024-09-18T01:25:20.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_41.status 2024-09-18T01:25:20.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:20.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.status 2024-09-18T01:25:20.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_41 2024-09-18T01:25:20.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:20.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:20.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:20.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_41 2024-09-18T01:25:21.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 166 2024-09-18T01:25:21.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_41 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_41.status 2024-09-18T01:25:21.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:21.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:21.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:21.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_41.status 2024-09-18T01:25:21.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_41 --format xml 2024-09-18T01:25:21.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:21.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:25:21.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=166 2024-09-18T01:25:21.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:21.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:21.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.status 2024-09-18T01:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:22.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:22.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.status 2024-09-18T01:25:22.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2024-09-18T01:25:22.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:22.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:25:22.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:25:22.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:25:22.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:25:22.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 166 = '' 2024-09-18T01:25:22.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:22.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:25:22.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.status 2024-09-18T01:25:22.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:22.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:22.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:22.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.status 2024-09-18T01:25:22.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2024-09-18T01:25:22.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:22.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:22.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:22.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:25:22.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:25:22.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:25:22.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:25:22.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 166 = '' 2024-09-18T01:25:22.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:22.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:25:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.status 2024-09-18T01:25:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.status 2024-09-18T01:25:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2024-09-18T01:25:23.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:23.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:23.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=166 2024-09-18T01:25:23.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 166 = 166 2024-09-18T01:25:23.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:23.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_41 snap 2024-09-18T01:25:23.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:23.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:23.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:23.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:23.374 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:23.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:25:23.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:24.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_41@snap 2024-09-18T01:25:24.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_41': 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bc4c8d9f5a 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bc4c8d9f5a 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:51 2024 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:51 2024 2024-09-18T01:25:24.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:51 2024 2024-09-18T01:25:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:25:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:25:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:25:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 0071b191-d397-4a11-b33d-d10f4477a5a3 2024-09-18T01:25:24.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:25:24.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:24.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_41 snap 2024-09-18T01:25:24.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:24.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:25:24.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:24.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:25:24.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_41.export 2024-09-18T01:25:24.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.export 2024-09-18T01:25:24.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_41.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.export 2024-09-18T01:25:24.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_41@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_41.export 2024-09-18T01:25:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:24.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_41@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.export 2024-09-18T01:25:24.797 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:24.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_41.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.export 2024-09-18T01:25:24.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_41.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_41.export 2024-09-18T01:25:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:25:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_42 2024-09-18T01:25:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_42 snap 2024-09-18T01:25:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:24.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_42@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:25:26.577 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_42 2024-09-18T01:25:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_42 started 2024-09-18T01:25:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:25:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:26.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_42 started 2024-09-18T01:25:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:25:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:25:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:25:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_42 2024-09-18T01:25:27.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:25:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:25:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:25:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:25:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:25:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:25:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:25:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:25:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_42 2024-09-18T01:25:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:25:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:25:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:25:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:27.584 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:25:27.585 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:25:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:25:27.586 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:25:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:27.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_42 2024-09-18T01:25:27.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-09-18T01:25:27.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:25:27.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:25:27.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:27.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_42 2024-09-18T01:25:27.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:25:27.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:27.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:27.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:25:27.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:25:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:25:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:25:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:25:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:25:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:25:27.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:25:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:27.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:27.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:27.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_42 2024-09-18T01:25:27.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-09-18T01:25:27.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:25:27.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:25:27.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:27.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_42 2024-09-18T01:25:27.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:25:27.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:25:27.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:25:27.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:25:27.805 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:25:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:25:27.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:25:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:25:27.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_42 2024-09-18T01:25:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-09-18T01:25:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:25:27.902 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:25:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_42 2024-09-18T01:25:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:25:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:25:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:25:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:27.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:25:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:25:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:25:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:25:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:25:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:25:27.907 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:25:27.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:25:27.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:25:27.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:25:27.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_42 2024-09-18T01:25:28.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:25:28.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:25:28.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:25:28.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:25:28.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:25:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:25:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:25:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_42 2024-09-18T01:25:28.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_42 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_42.status 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_42.status 2024-09-18T01:25:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:28.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_42.status 2024-09-18T01:25:28.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_42.status 2024-09-18T01:25:28.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_42.status 2024-09-18T01:25:28.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:28.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.status 2024-09-18T01:25:28.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_42 2024-09-18T01:25:28.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:28.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:28.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:28.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_42 2024-09-18T01:25:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 168 2024-09-18T01:25:28.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_42 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_42.status 2024-09-18T01:25:28.589 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:28.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:28.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:28.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_42.status 2024-09-18T01:25:28.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_42 --format xml 2024-09-18T01:25:28.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:28.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:25:28.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=168 2024-09-18T01:25:28.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:28.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:28.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:25:28.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.status 2024-09-18T01:25:28.836 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:28.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:28.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:28.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.status 2024-09-18T01:25:28.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2024-09-18T01:25:28.837 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:28.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=76 2024-09-18T01:25:28.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 168 = 76 2024-09-18T01:25:28.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:28.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:25:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.status 2024-09-18T01:25:29.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.status 2024-09-18T01:25:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2024-09-18T01:25:29.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:29.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:29.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:29.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:25:29.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:25:29.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:25:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:25:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 168 = '' 2024-09-18T01:25:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:25:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.status 2024-09-18T01:25:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.status 2024-09-18T01:25:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2024-09-18T01:25:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:30.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=168 2024-09-18T01:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 168 = 168 2024-09-18T01:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_42 snap 2024-09-18T01:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:25:30.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:31.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_42@snap 2024-09-18T01:25:31.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_42': 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 11c3749fca02 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.11c3749fca02 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:16:59 2024 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:16:59 2024 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:16:59 2024 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:25:31.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:25:31.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: aeaeb058-01f1-4690-999c-8c21dad1561e 2024-09-18T01:25:31.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:25:31.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:31.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_42 snap 2024-09-18T01:25:31.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:31.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:25:31.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:31.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:25:31.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_42.export 2024-09-18T01:25:31.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.export 2024-09-18T01:25:31.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_42.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.export 2024-09-18T01:25:31.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_42@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_42.export 2024-09-18T01:25:31.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:31.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_42@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.export 2024-09-18T01:25:31.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:31.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_42.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.export 2024-09-18T01:25:31.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_42.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_42.export 2024-09-18T01:25:31.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:31.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:25:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_43 2024-09-18T01:25:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_43 snap 2024-09-18T01:25:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:31.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_43@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_43 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_43 started 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_43 started 2024-09-18T01:25:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:25:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:25:34.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:25:34.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_43 2024-09-18T01:25:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:25:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:25:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:25:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:25:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:25:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:25:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:25:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:25:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:34.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:34.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:25:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:25:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:25:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:25:34.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:25:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:34.603 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:34.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:34.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:25:34.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:25:34.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:25:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:34.623 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2024-09-18T01:25:34.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2024-09-18T01:25:34.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:25:34.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:25:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2024-09-18T01:25:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:25:34.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:25:34.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:25:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:25:34.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:25:34.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:34.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:34.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:25:34.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2024-09-18T01:25:34.815 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2024-09-18T01:25:34.815 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:25:34.815 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:25:34.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:34.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2024-09-18T01:25:34.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:25:34.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:34.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:34.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:25:34.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:25:34.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:34.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:34.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:34.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:25:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:25:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:25:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:25:34.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:25:34.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:25:34.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:25:34.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:25:34.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2024-09-18T01:25:34.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:25:34.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:25:34.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:25:34.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:25:34.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:25:34.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_43 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_43 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_43.status 2024-09-18T01:25:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_43.status 2024-09-18T01:25:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_43.status 2024-09-18T01:25:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_43.status 2024-09-18T01:25:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_43.status 2024-09-18T01:25:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.status 2024-09-18T01:25:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_43 2024-09-18T01:25:34.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:34.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:34.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:34.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_43 2024-09-18T01:25:35.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 170 2024-09-18T01:25:35.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_43 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_43.status 2024-09-18T01:25:35.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:35.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:35.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:35.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_43.status 2024-09-18T01:25:35.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_43 --format xml 2024-09-18T01:25:35.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:35.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:25:35.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=170 2024-09-18T01:25:35.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:35.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:35.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:25:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.status 2024-09-18T01:25:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:35.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.status 2024-09-18T01:25:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2024-09-18T01:25:35.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:35.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=77 2024-09-18T01:25:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 170 = 77 2024-09-18T01:25:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:35.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:25:36.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.status 2024-09-18T01:25:36.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:36.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:36.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:36.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.status 2024-09-18T01:25:36.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2024-09-18T01:25:36.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:36.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:25:36.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:25:36.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:25:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:25:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 170 = '' 2024-09-18T01:25:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:36.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:25:37.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.status 2024-09-18T01:25:37.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:37.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:37.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:37.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.status 2024-09-18T01:25:37.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2024-09-18T01:25:37.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:37.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:37.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=170 2024-09-18T01:25:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 170 = 170 2024-09-18T01:25:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_43 snap 2024-09-18T01:25:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:25:37.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:38.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_43@snap 2024-09-18T01:25:38.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_43': 2024-09-18T01:25:38.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:25:38.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:25:38.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:25:38.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 1268110236cf 2024-09-18T01:25:38.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.1268110236cf 2024-09-18T01:25:38.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:25:38.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:25:38.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:25:38.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:25:38.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:17:13 2024 2024-09-18T01:25:38.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:17:13 2024 2024-09-18T01:25:38.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:17:13 2024 2024-09-18T01:25:38.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:25:38.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:25:38.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:25:38.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: bf14a2b6-1625-4fba-8a13-3e69c7467ad0 2024-09-18T01:25:38.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:25:38.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:38.441 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_43 snap 2024-09-18T01:25:38.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:38.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:25:38.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:38.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:25:38.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_43.export 2024-09-18T01:25:38.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.export 2024-09-18T01:25:38.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_43.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.export 2024-09-18T01:25:38.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_43@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_43.export 2024-09-18T01:25:38.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:38.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_43@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.export 2024-09-18T01:25:38.669 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:38.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_43.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.export 2024-09-18T01:25:38.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_43.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_43.export 2024-09-18T01:25:38.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:38.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:25:38.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_44 2024-09-18T01:25:38.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_44 snap 2024-09-18T01:25:38.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:38.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:38.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:38.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:38.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_44@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:25:40.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_44 2024-09-18T01:25:40.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:40.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:40.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:40.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_44 started 2024-09-18T01:25:40.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:40.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:40.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:40.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:25:40.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:40.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:40.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:41.464 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7515242896052667) > weight (0.25) 2024-09-18T01:25:41.464 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9959524576943981) > weight (0.25) 2024-09-18T01:25:41.464 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5490279661731182) > weight (0.25) 2024-09-18T01:25:41.464 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.9096878045869353) > weight (0.25) 2024-09-18T01:25:41.464 INFO:tasks.rbd_mirror_thrash:waiting for 88 secs before thrashing 2024-09-18T01:25:41.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_44 started 2024-09-18T01:25:41.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:41.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:41.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:41.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:25:41.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:25:41.607 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:25:41.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_44 2024-09-18T01:25:41.608 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:25:41.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:25:41.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:25:41.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:41.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_44 2024-09-18T01:25:41.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:25:41.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:41.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:41.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:41.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:41.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:25:41.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:25:41.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:41.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:41.613 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:25:41.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:25:41.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:41.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:41.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:41.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_44 2024-09-18T01:25:41.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:25:41.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:25:41.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:25:41.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:25:41.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:25:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:25:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:25:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:41.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_44 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_44 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_44.status 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_44.status 2024-09-18T01:25:41.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:41.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_44.status 2024-09-18T01:25:41.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_44.status 2024-09-18T01:25:41.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_44.status 2024-09-18T01:25:41.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:41.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.status 2024-09-18T01:25:41.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_44 2024-09-18T01:25:41.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:41.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:41.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:41.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_44 2024-09-18T01:25:42.615 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 172 2024-09-18T01:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_44 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_44.status 2024-09-18T01:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_44.status 2024-09-18T01:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_44 --format xml 2024-09-18T01:25:42.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:42.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:25:42.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=172 2024-09-18T01:25:42.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:42.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:42.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:25:42.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.status 2024-09-18T01:25:42.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:42.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:42.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:42.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.status 2024-09-18T01:25:42.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2024-09-18T01:25:42.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:42.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=78 2024-09-18T01:25:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 172 = 78 2024-09-18T01:25:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:25:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.status 2024-09-18T01:25:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.status 2024-09-18T01:25:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2024-09-18T01:25:43.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:43.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:43.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:25:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:25:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:25:43.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:25:43.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 172 = '' 2024-09-18T01:25:43.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:43.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:25:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.status 2024-09-18T01:25:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.status 2024-09-18T01:25:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2024-09-18T01:25:44.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:44.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:44.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=172 2024-09-18T01:25:44.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 172 = 172 2024-09-18T01:25:44.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:44.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_44 snap 2024-09-18T01:25:44.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:44.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:44.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:44.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:44.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:44.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:25:44.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:45.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_44@snap 2024-09-18T01:25:45.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_44': 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 129dd140858 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.129dd140858 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:17:27 2024 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:17:27 2024 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:17:27 2024 2024-09-18T01:25:45.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:25:45.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:25:45.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:25:45.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: f0afae9f-39f5-4655-a91e-7d6dfba2149c 2024-09-18T01:25:45.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:25:45.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:45.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_44 snap 2024-09-18T01:25:45.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:45.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:25:45.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:45.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:25:45.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_44.export 2024-09-18T01:25:45.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.export 2024-09-18T01:25:45.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_44.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.export 2024-09-18T01:25:45.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_44@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_44.export 2024-09-18T01:25:45.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:45.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_44@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.export 2024-09-18T01:25:45.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:45.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_44.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.export 2024-09-18T01:25:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_44.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_44.export 2024-09-18T01:25:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:25:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_45 2024-09-18T01:25:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_45 snap 2024-09-18T01:25:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:45.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_45@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:25:47.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_45 2024-09-18T01:25:47.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:47.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:47.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:47.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_45 started 2024-09-18T01:25:47.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:47.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:47.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:47.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:25:47.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:47.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:47.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_45 started 2024-09-18T01:25:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:25:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:25:48.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:25:48.636 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_45 2024-09-18T01:25:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:25:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:25:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:25:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:25:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:25:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:25:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:25:48.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:25:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_45 2024-09-18T01:25:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:25:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:25:48.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:25:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:25:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:25:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:48.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:25:48.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:25:48.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:25:48.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:48.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:48.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:48.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_45 2024-09-18T01:25:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:25:48.762 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:25:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:25:48.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:25:48.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_45 2024-09-18T01:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:48.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:25:48.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:25:48.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_45 2024-09-18T01:25:48.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:48.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:48.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:48.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:48.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_45.status 2024-09-18T01:25:48.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_45.status 2024-09-18T01:25:48.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:48.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_45.status 2024-09-18T01:25:48.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_45.status 2024-09-18T01:25:48.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_45.status 2024-09-18T01:25:48.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:48.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.status 2024-09-18T01:25:48.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_45 2024-09-18T01:25:48.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:48.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:48.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:48.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_45 2024-09-18T01:25:49.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 174 2024-09-18T01:25:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_45 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_45.status 2024-09-18T01:25:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:49.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_45.status 2024-09-18T01:25:49.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_45 --format xml 2024-09-18T01:25:49.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:49.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:25:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=174 2024-09-18T01:25:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:49.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:25:49.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.status 2024-09-18T01:25:49.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:49.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:49.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:49.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.status 2024-09-18T01:25:49.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2024-09-18T01:25:49.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:49.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:49.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=79 2024-09-18T01:25:49.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 174 = 79 2024-09-18T01:25:49.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:49.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:25:50.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.status 2024-09-18T01:25:50.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:50.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:50.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:50.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.status 2024-09-18T01:25:50.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2024-09-18T01:25:50.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:50.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:50.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:50.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:25:50.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:25:50.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:25:50.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:25:50.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 174 = '' 2024-09-18T01:25:50.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:50.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:25:51.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.status 2024-09-18T01:25:51.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:51.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:51.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:51.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.status 2024-09-18T01:25:51.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2024-09-18T01:25:51.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:51.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=174 2024-09-18T01:25:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 174 = 174 2024-09-18T01:25:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:51.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_45 snap 2024-09-18T01:25:51.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:51.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:51.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:51.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:51.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:51.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:25:51.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:52.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_45@snap 2024-09-18T01:25:52.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_45': 2024-09-18T01:25:52.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:25:52.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:25:52.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:25:52.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 129defd8f163 2024-09-18T01:25:52.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.129defd8f163 2024-09-18T01:25:52.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:25:52.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:25:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:25:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:25:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:17:36 2024 2024-09-18T01:25:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:17:36 2024 2024-09-18T01:25:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:17:36 2024 2024-09-18T01:25:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:25:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:25:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:25:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 4858f75f-8426-4450-aea4-9c743ab91e23 2024-09-18T01:25:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:25:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_45 snap 2024-09-18T01:25:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:25:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:25:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_45.export 2024-09-18T01:25:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.export 2024-09-18T01:25:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_45.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.export 2024-09-18T01:25:52.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_45@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_45.export 2024-09-18T01:25:52.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:52.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_45@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.export 2024-09-18T01:25:52.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:52.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_45.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.export 2024-09-18T01:25:52.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_45.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_45.export 2024-09-18T01:25:52.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:52.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:25:52.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_46 2024-09-18T01:25:52.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_46 snap 2024-09-18T01:25:52.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:52.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:52.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:52.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:52.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_46@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:25:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_46 2024-09-18T01:25:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_46 started 2024-09-18T01:25:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:25:54.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:54.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:54.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:55.654 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_46 started 2024-09-18T01:25:55.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:55.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:55.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:55.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:25:55.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:25:55.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:25:55.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_46 2024-09-18T01:25:55.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:25:55.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:25:55.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:55.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:25:55.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:25:55.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:25:55.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:25:55.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:25:55.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:25:55.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:25:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:25:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2024-09-18T01:25:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:25:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:25:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:25:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:25:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:25:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:25:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:25:55.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:25:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:25:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:25:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:25:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:25:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:25:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:25:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:25:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:25:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:25:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:25:55.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2024-09-18T01:25:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:25:55.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:25:55.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:25:55.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:25:55.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:25:55.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:25:55.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:25:55.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:55.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_46 2024-09-18T01:25:55.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:55.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:55.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:55.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:55.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:25:55.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:25:55.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_46 2024-09-18T01:25:55.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:25:55.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:55.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:55.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:55.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_46.status 2024-09-18T01:25:55.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_46.status 2024-09-18T01:25:55.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:55.783 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_46.status 2024-09-18T01:25:55.783 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_46.status 2024-09-18T01:25:55.783 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_46.status 2024-09-18T01:25:55.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:25:55.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.status 2024-09-18T01:25:55.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_46 2024-09-18T01:25:55.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:55.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:55.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:55.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_46 2024-09-18T01:25:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 176 2024-09-18T01:25:56.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_46 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_46.status 2024-09-18T01:25:56.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:56.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:56.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:56.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_46.status 2024-09-18T01:25:56.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_46 --format xml 2024-09-18T01:25:56.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:56.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:25:56.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=176 2024-09-18T01:25:56.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:56.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:56.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:25:56.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.status 2024-09-18T01:25:56.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:56.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:56.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:56.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.status 2024-09-18T01:25:56.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2024-09-18T01:25:56.918 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:56.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:56.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=80 2024-09-18T01:25:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 176 = 80 2024-09-18T01:25:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:56.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:25:57.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.status 2024-09-18T01:25:57.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:57.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:57.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:57.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.status 2024-09-18T01:25:57.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2024-09-18T01:25:57.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:57.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:25:57.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:57.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:25:57.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:25:57.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:25:57.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:25:57.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 176 = '' 2024-09-18T01:25:57.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:25:57.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:25:58.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.status 2024-09-18T01:25:58.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:58.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:58.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:58.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.status 2024-09-18T01:25:58.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2024-09-18T01:25:58.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:25:58.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:25:58.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=176 2024-09-18T01:25:58.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 176 = 176 2024-09-18T01:25:58.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:58.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_46 snap 2024-09-18T01:25:58.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:25:58.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:58.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:58.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:58.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:25:58.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:25:58.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:25:59.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_46@snap 2024-09-18T01:25:59.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_46': 2024-09-18T01:25:59.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:25:59.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:25:59.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:25:59.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 129dd345da8c 2024-09-18T01:25:59.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.129dd345da8c 2024-09-18T01:25:59.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:25:59.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:25:59.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:25:59.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:25:59.313 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:17:44 2024 2024-09-18T01:25:59.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:17:44 2024 2024-09-18T01:25:59.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:17:44 2024 2024-09-18T01:25:59.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:25:59.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:25:59.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:25:59.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 48442b3f-58cc-4f8f-8a64-4a412ad18406 2024-09-18T01:25:59.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:25:59.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:59.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_46 snap 2024-09-18T01:25:59.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:59.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:25:59.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:59.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:25:59.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_46.export 2024-09-18T01:25:59.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.export 2024-09-18T01:25:59.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_46.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.export 2024-09-18T01:25:59.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_46@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_46.export 2024-09-18T01:25:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:59.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_46@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.export 2024-09-18T01:25:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:25:59.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_46.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.export 2024-09-18T01:25:59.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_46.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_46.export 2024-09-18T01:25:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:25:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:25:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_47 2024-09-18T01:25:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_47 snap 2024-09-18T01:25:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:25:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:25:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:25:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:25:59.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_47@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:26:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_47 2024-09-18T01:26:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_47 started 2024-09-18T01:26:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:26:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:26:01.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:26:01.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:26:02.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_47 started 2024-09-18T01:26:02.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:02.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:02.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:02.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:26:02.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:26:02.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:26:02.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_47 2024-09-18T01:26:02.678 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:26:02.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:02.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:26:02.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:26:02.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:26:02.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:26:02.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:26:02.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:26:02.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:26:02.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:26:02.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:26:02.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_47 2024-09-18T01:26:02.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:26:02.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:02.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:26:02.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:26:02.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:26:02.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:26:02.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:26:02.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:26:02.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:26:02.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:26:02.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:26:02.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:02.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:26:02.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:26:02.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:26:02.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:26:02.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:26:02.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:26:02.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:26:02.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:26:02.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:26:02.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:26:02.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:02.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:26:02.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:26:02.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:26:02.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:26:02.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:26:02.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:26:02.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:26:02.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:02.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:02.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:02.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_47 2024-09-18T01:26:02.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2024-09-18T01:26:02.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:26:02.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:26:02.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:26:02.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_47 2024-09-18T01:26:02.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:26:02.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:02.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:26:02.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:26:02.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:26:02.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:26:02.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:26:02.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:26:02.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:26:02.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:26:02.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:02.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:26:02.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:26:02.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:26:02.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:26:02.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:26:02.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:26:02.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:26:02.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:26:02.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:26:02.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:26:02.811 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_47 2024-09-18T01:26:02.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2024-09-18T01:26:02.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:26:02.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:26:02.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:26:02.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_47 2024-09-18T01:26:02.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:26:02.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:02.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:26:02.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:26:02.898 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:26:02.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:26:02.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:26:02.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:26:02.915 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_47 2024-09-18T01:26:03.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:26:03.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:26:03.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:26:03.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:26:03.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_47 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_47 2024-09-18T01:26:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:26:03.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:03.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:03.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:03.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_47.status 2024-09-18T01:26:03.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_47.status 2024-09-18T01:26:03.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:26:03.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_47.status 2024-09-18T01:26:03.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_47.status 2024-09-18T01:26:03.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_47.status 2024-09-18T01:26:03.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:26:03.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.status 2024-09-18T01:26:03.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_47 2024-09-18T01:26:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:03.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_47 2024-09-18T01:26:03.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 178 2024-09-18T01:26:03.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_47 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_47.status 2024-09-18T01:26:03.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:03.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:03.687 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:03.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_47.status 2024-09-18T01:26:03.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_47 --format xml 2024-09-18T01:26:03.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:03.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:26:03.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=178 2024-09-18T01:26:03.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:03.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:03.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:26:03.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.status 2024-09-18T01:26:03.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:03.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:03.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:03.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.status 2024-09-18T01:26:03.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2024-09-18T01:26:03.935 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:03.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:03.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:03.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:26:03.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:26:03.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:26:03.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:26:03.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 178 = '' 2024-09-18T01:26:03.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:03.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:26:04.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.status 2024-09-18T01:26:04.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:04.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:04.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:04.385 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.status 2024-09-18T01:26:04.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2024-09-18T01:26:04.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:04.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:04.433 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:04.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:26:04.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:26:04.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:26:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:26:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 178 = '' 2024-09-18T01:26:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:04.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:26:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.status 2024-09-18T01:26:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.status 2024-09-18T01:26:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2024-09-18T01:26:05.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:05.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=178 2024-09-18T01:26:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 178 = 178 2024-09-18T01:26:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_47 snap 2024-09-18T01:26:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:26:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:26:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:26:05.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:26:06.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_47@snap 2024-09-18T01:26:06.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_47': 2024-09-18T01:26:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:26:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:26:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:26:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 12683e407693 2024-09-18T01:26:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.12683e407693 2024-09-18T01:26:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:26:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:26:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:26:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:26:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:18:00 2024 2024-09-18T01:26:06.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:18:00 2024 2024-09-18T01:26:06.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:18:00 2024 2024-09-18T01:26:06.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:26:06.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:26:06.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:26:06.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: dc5f334a-10d4-4c3c-9345-0b305199e1f2 2024-09-18T01:26:06.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:26:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_47 snap 2024-09-18T01:26:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:26:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:26:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:26:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_47.export 2024-09-18T01:26:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.export 2024-09-18T01:26:06.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_47.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.export 2024-09-18T01:26:06.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_47@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_47.export 2024-09-18T01:26:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:26:06.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_47@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.export 2024-09-18T01:26:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:26:06.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_47.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.export 2024-09-18T01:26:06.806 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_47.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_47.export 2024-09-18T01:26:06.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:06.838 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_48 2024-09-18T01:26:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_48 snap 2024-09-18T01:26:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:26:06.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_48@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_48 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_48 started 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:26:08.694 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:26:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_48 started 2024-09-18T01:26:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:26:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:26:09.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:26:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_48 2024-09-18T01:26:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:26:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:26:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:26:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:26:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:26:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:26:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:26:09.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:26:09.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:26:09.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:26:09.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_48 2024-09-18T01:26:09.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:26:09.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:09.700 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:26:09.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:26:09.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:26:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:09.722 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_48 2024-09-18T01:26:09.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:26:09.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:26:09.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:26:09.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:26:09.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:26:09.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:26:09.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:26:09.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_48 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_48 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:09.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:09.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_48.status 2024-09-18T01:26:09.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_48.status 2024-09-18T01:26:09.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:26:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_48.status 2024-09-18T01:26:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_48.status 2024-09-18T01:26:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_48.status 2024-09-18T01:26:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:26:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.status 2024-09-18T01:26:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_48 2024-09-18T01:26:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:09.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_48 2024-09-18T01:26:10.705 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 180 2024-09-18T01:26:10.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_48 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_48.status 2024-09-18T01:26:10.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:10.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:10.715 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:10.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_48.status 2024-09-18T01:26:10.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_48 --format xml 2024-09-18T01:26:10.716 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:10.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:26:10.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=180 2024-09-18T01:26:10.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:10.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:10.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:26:10.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.status 2024-09-18T01:26:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.status 2024-09-18T01:26:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2024-09-18T01:26:10.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:11.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:11.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:11.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:26:11.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:26:11.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:26:11.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:26:11.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 180 = '' 2024-09-18T01:26:11.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:11.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:26:11.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.status 2024-09-18T01:26:11.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:11.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:11.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.status 2024-09-18T01:26:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2024-09-18T01:26:11.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:11.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:11.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:11.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:26:11.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:26:11.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:26:11.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:26:11.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 180 = '' 2024-09-18T01:26:11.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:11.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:26:12.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.status 2024-09-18T01:26:12.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:12.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:12.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:12.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.status 2024-09-18T01:26:12.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2024-09-18T01:26:12.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:12.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:12.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=180 2024-09-18T01:26:12.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 180 = 180 2024-09-18T01:26:12.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:12.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_48 snap 2024-09-18T01:26:12.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:26:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:26:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:26:12.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:26:13.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_48@snap 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_48': 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 129d5d1cc97b 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.129d5d1cc97b 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:18:13 2024 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:18:13 2024 2024-09-18T01:26:13.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:18:13 2024 2024-09-18T01:26:13.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:26:13.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:26:13.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:26:13.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: db673391-3699-4385-93be-31f993e37bad 2024-09-18T01:26:13.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:26:13.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:13.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_48 snap 2024-09-18T01:26:13.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:13.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:26:13.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:26:13.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:26:13.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_48.export 2024-09-18T01:26:13.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.export 2024-09-18T01:26:13.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_48.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.export 2024-09-18T01:26:13.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_48@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_48.export 2024-09-18T01:26:13.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:26:13.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_48@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.export 2024-09-18T01:26:13.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:26:13.756 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_48.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.export 2024-09-18T01:26:13.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_48.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_48.export 2024-09-18T01:26:13.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:13.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:13.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_49 2024-09-18T01:26:13.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_49 snap 2024-09-18T01:26:13.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:13.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:13.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:13.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:26:13.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_49@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:26:15.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_49 2024-09-18T01:26:15.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:15.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:15.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:15.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_49 started 2024-09-18T01:26:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:26:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:26:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:26:15.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:26:15.783 INFO:tasks.ceph.cluster1.osd.2.smithi078.stderr:2024-09-18T01:26:15.782+0000 7f433a1e4640 -1 reset not still connected to 0x5577e72daa90 2024-09-18T01:26:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_49 started 2024-09-18T01:26:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_49 2024-09-18T01:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:26:16.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:26:16.741 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:26:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:26:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:26:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:26:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:26:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:26:16.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:16.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2024-09-18T01:26:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-18T01:26:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:26:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:26:16.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:26:16.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2024-09-18T01:26:16.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:26:16.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:16.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:26:16.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:26:16.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:26:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:26:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:26:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:26:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:26:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:26:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:26:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:26:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:26:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:26:16.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:26:16.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:26:16.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:26:16.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:26:16.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2024-09-18T01:26:16.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-09-18T01:26:16.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:26:16.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:26:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:26:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2024-09-18T01:26:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:26:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:26:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:26:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:26:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:26:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:26:16.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:26:16.959 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:26:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:26:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:26:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:26:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:26:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:26:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:26:16.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:26:16.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:26:16.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:26:16.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:26:16.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2024-09-18T01:26:17.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:26:17.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:26:17.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:26:17.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:26:17.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:26:17.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:26:17.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:26:17.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:17.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_49 2024-09-18T01:26:17.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:26:17.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:17.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:17.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:17.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:26:17.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:26:17.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_49 2024-09-18T01:26:17.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:26:17.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:17.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:17.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:17.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_49.status 2024-09-18T01:26:17.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_49.status 2024-09-18T01:26:17.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:26:17.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_49.status 2024-09-18T01:26:17.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_49.status 2024-09-18T01:26:17.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_49.status 2024-09-18T01:26:17.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:26:17.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.status 2024-09-18T01:26:17.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_49 2024-09-18T01:26:17.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:17.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:17.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:17.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_49 2024-09-18T01:26:17.720 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 182 2024-09-18T01:26:17.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_49 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_49.status 2024-09-18T01:26:17.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:17.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:17.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_49.status 2024-09-18T01:26:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_49 --format xml 2024-09-18T01:26:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:17.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:26:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=182 2024-09-18T01:26:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:17.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:26:17.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.status 2024-09-18T01:26:17.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:17.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:17.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:17.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.status 2024-09-18T01:26:17.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2024-09-18T01:26:17.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:18.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:18.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:18.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:26:18.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:26:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:26:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:26:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 182 = '' 2024-09-18T01:26:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:26:18.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.status 2024-09-18T01:26:18.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:18.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:18.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:18.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.status 2024-09-18T01:26:18.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2024-09-18T01:26:18.430 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:18.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:26:18.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:26:18.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:26:18.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:26:18.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 182 = '' 2024-09-18T01:26:18.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:18.482 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:26:19.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.status 2024-09-18T01:26:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.status 2024-09-18T01:26:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2024-09-18T01:26:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:19.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:19.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=182 2024-09-18T01:26:19.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 182 = 182 2024-09-18T01:26:19.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:19.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_49 snap 2024-09-18T01:26:19.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:19.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:19.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:19.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:26:19.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:26:19.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:26:19.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:26:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_49@snap 2024-09-18T01:26:20.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_49': 2024-09-18T01:26:20.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:26:20.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:26:20.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:26:20.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 10bce25a1958 2024-09-18T01:26:20.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.10bce25a1958 2024-09-18T01:26:20.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:26:20.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:26:20.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:26:20.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:26:20.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:18:25 2024 2024-09-18T01:26:20.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:18:25 2024 2024-09-18T01:26:20.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:18:25 2024 2024-09-18T01:26:20.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:26:20.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:26:20.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:26:20.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 380bbc6e-0c06-47e3-a103-ff2ba6ff04d9 2024-09-18T01:26:20.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:26:20.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_49 snap 2024-09-18T01:26:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:26:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:26:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:26:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_49.export 2024-09-18T01:26:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.export 2024-09-18T01:26:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_49.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.export 2024-09-18T01:26:20.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_49@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_49.export 2024-09-18T01:26:20.559 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:26:20.568 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_49@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.export 2024-09-18T01:26:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:26:20.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_49.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.export 2024-09-18T01:26:20.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_49.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_49.export 2024-09-18T01:26:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_50 2024-09-18T01:26:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_snap cluster2 mirror image_50 snap 2024-09-18T01:26:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:26:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror snap create image_50@snap --debug-rbd=20 --debug-journaler=20 2024-09-18T01:26:22.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_started cluster1 mirror image_50 2024-09-18T01:26:22.727 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:22.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:22.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:22.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_replay_state cluster1 mirror image_50 started 2024-09-18T01:26:22.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:22.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:22.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:22.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=started 2024-09-18T01:26:22.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:26:22.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:26:22.728 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:26:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_replay_state cluster1 mirror image_50 started 2024-09-18T01:26:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=started 2024-09-18T01:26:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_result 2024-09-18T01:26:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=stopped 2024-09-18T01:26:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_50 2024-09-18T01:26:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -i state 2024-09-18T01:26:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_instance=cluster1 2024-09-18T01:26:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:26:23.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance=cluster1 2024-09-18T01:26:23.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local loop_instance 2024-09-18T01:26:23.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-09-18T01:26:23.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sleep 0 2024-09-18T01:26:23.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ seq 0 3 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_50 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:0 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:0 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:26:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:26:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 0 = cluster1:0 ']' 2024-09-18T01:26:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:26:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=0 2024-09-18T01:26:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:26:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-09-18T01:26:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:0 2024-09-18T01:26:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:26:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:26:23.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-09-18T01:26:23.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:0 2024-09-18T01:26:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:26:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:26:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=0 2024-09-18T01:26:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 0 = cluster1:0 ']' 2024-09-18T01:26:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:26:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:26:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=0 2024-09-18T01:26:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=0 2024-09-18T01:26:23.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-09-18T01:26:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-09-18T01:26:23.755 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_50 2024-09-18T01:26:23.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-18T01:26:23.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:26:23.845 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:26:23.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_50 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:1 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:1 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=1 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 1 = cluster1:1 ']' 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:23.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=1 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:1 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:1 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=1 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 1 = cluster1:1 ']' 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:26:23.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:26:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=1 2024-09-18T01:26:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=1 2024-09-18T01:26:23.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-09-18T01:26:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:26:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:26:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-09-18T01:26:23.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_50 2024-09-18T01:26:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-18T01:26:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:26:23.954 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_50 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:2 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:2 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=2 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 2 = cluster1:2 ']' 2024-09-18T01:26:23.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=2 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:2 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:2 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=2 2024-09-18T01:26:23.956 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 2 = cluster1:2 ']' 2024-09-18T01:26:23.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:26:23.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:26:23.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=2 2024-09-18T01:26:23.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=2 2024-09-18T01:26:23.957 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-09-18T01:26:23.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:26:23.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:26:23.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-09-18T01:26:23.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_50 2024-09-18T01:26:24.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-09-18T01:26:24.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd mirror status : get status for rbd mirror 2024-09-18T01:26:24.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:admin_socket: invalid command 2024-09-18T01:26:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:26:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_50 2024-09-18T01:26:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1:3 2024-09-18T01:26:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ shift 2024-09-18T01:26:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance 2024-09-18T01:26:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-09-18T01:26:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local val=cluster1:3 2024-09-18T01:26:24.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster_var_name=cluster 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local instance_var_name=instance 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=3 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ '[' 3 = cluster1:3 ']' 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval cluster=cluster1 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ eval instance=3 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local local_cluster=cluster1:3 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster=cluster1 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local val=cluster1:3 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local cluster_var_name=local_cluster 2024-09-18T01:26:24.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ local instance_var_name=instance 2024-09-18T01:26:24.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ cluster=cluster1 2024-09-18T01:26:24.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ instance=3 2024-09-18T01:26:24.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ '[' 3 = cluster1:3 ']' 2024-09-18T01:26:24.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval local_cluster=cluster1 2024-09-18T01:26:24.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ local_cluster=cluster1 2024-09-18T01:26:24.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ eval instance=3 2024-09-18T01:26:24.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ instance=3 2024-09-18T01:26:24.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-09-18T01:26:24.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:26:24.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:26:24.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-09-18T01:26:24.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_50 2024-09-18T01:26:24.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ return 0 2024-09-18T01:26:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ status_result=' "state": "Replaying"' 2024-09-18T01:26:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo ' "state": "Replaying"' 2024-09-18T01:26:24.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ grep -i Replaying 2024-09-18T01:26:24.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: "state": "Replaying" 2024-09-18T01:26:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=started 2024-09-18T01:26:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test started = started 2024-09-18T01:26:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_50 2024-09-18T01:26:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:26:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:24.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:24.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:24.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = journal ']' 2024-09-18T01:26:24.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' snapshot = snapshot ']' 2024-09-18T01:26:24.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_50 2024-09-18T01:26:24.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_cluster=cluster1 2024-09-18T01:26:24.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:24.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:24.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:24.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster2-mirror-image_50.status 2024-09-18T01:26:24.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster2-mirror-image_50.status 2024-09-18T01:26:24.169 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:26:24.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local status_log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_50.status 2024-09-18T01:26:24.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ mkfname cluster1-mirror-image_50.status 2024-09-18T01:26:24.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ echo cluster1-mirror-image_50.status 2024-09-18T01:26:24.170 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -e 's|[/ ]|_|g' 2024-09-18T01:26:24.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local local_status_log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.status 2024-09-18T01:26:24.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ mirror_image_snapshot cluster2 mirror image_50 2024-09-18T01:26:24.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:24.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:24.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:24.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_50 2024-09-18T01:26:24.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Snapshot ID: 184 2024-09-18T01:26:24.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_50 /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_50.status 2024-09-18T01:26:24.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:24.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:24.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:24.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_50.status 2024-09-18T01:26:24.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_50 --format xml 2024-09-18T01:26:24.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:24.782 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-09-18T01:26:24.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snapshot_id=184 2024-09-18T01:26:24.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:24.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:24.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.2 2024-09-18T01:26:24.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.status 2024-09-18T01:26:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.status 2024-09-18T01:26:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2024-09-18T01:26:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:25.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:25.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:25.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:26:25.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:26:25.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:26:25.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:26:25.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 184 = '' 2024-09-18T01:26:25.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:25.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.4 2024-09-18T01:26:25.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.status 2024-09-18T01:26:25.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:25.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:25.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:25.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.status 2024-09-18T01:26:25.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2024-09-18T01:26:25.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:25.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:26:25.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:-:1.1: Document is empty 2024-09-18T01:26:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: 2024-09-18T01:26:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:^ 2024-09-18T01:26:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id= 2024-09-18T01:26:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 184 = '' 2024-09-18T01:26:25.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-09-18T01:26:25.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.8 2024-09-18T01:26:26.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.status 2024-09-18T01:26:26.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:26.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:26.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:26.285 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local log=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.status 2024-09-18T01:26:26.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2024-09-18T01:26:26.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-09-18T01:26:26.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-09-18T01:26:26.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local primary_snapshot_id=184 2024-09-18T01:26:26.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 184 = 184 2024-09-18T01:26:26.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:26.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_snap_present cluster1 mirror image_50 snap 2024-09-18T01:26:26.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:26:26.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:26.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:26.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:26:26.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:26:26.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-09-18T01:26:26.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 1 2024-09-18T01:26:27.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 info mirror/image_50@snap 2024-09-18T01:26:27.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_50': 2024-09-18T01:26:27.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 128 MiB in 32 objects 2024-09-18T01:26:27.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:26:27.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 2 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 11c39306a671 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.11c39306a671 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock, non-primary 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:18:37 2024 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:18:37 2024 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:18:37 2024 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: protected: False 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:26:27.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: e589d353-55de-4143-8072-8a9ace421dbf 2024-09-18T01:26:27.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: false 2024-09-18T01:26:27.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:27.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ compare_image_snaps mirror image_50 snap 2024-09-18T01:26:27.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:27.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:26:27.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap_name=snap 2024-09-18T01:26:27.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local ret=0 2024-09-18T01:26:27.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local rmt_export=/tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_50.export 2024-09-18T01:26:27.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local loc_export=/tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.export 2024-09-18T01:26:27.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_50.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.export 2024-09-18T01:26:27.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 -p mirror export image_50@snap /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_50.export 2024-09-18T01:26:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:26:27.552 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster1 -p mirror export image_50@snap /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.export 2024-09-18T01:26:27.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-09-18T01:26:27.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cmp /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_50.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.export 2024-09-18T01:26:27.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -f /tmp/tmp.yeNdCEAq6i/cluster2-mirror-image_50.export /tmp/tmp.yeNdCEAq6i/cluster1-mirror-image_50.export 2024-09-18T01:26:27.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ testlog 'TEST: delete many images' 2024-09-18T01:26:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ tee -a /tmp/tmp.yeNdCEAq6i/rbd-mirror.test.log 2024-09-18T01:26:27.860 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ date '+%F %T' 2024-09-18T01:26:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo 2024-09-18 01:26:27 TEST: delete many images 2024-09-18T01:26:27.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:2024-09-18 01:26:27 TEST: delete many images 2024-09-18T01:26:27.874 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 50 2024-09-18T01:26:27.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:27.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_1 2024-09-18T01:26:27.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_1 snap 2024-09-18T01:26:27.875 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:26:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:27.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_1@snap 2024-09-18T01:26:28.740 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_1 2024-09-18T01:26:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:26:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:28.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_1 2024-09-18T01:26:28.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:28.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:28.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:26:28.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2024-09-18T01:26:28.800 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2024-09-18T01:26:28.907 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:26:28.906+0000 7ffa51edc640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cced9bb1e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:28.907 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:26:28.906+0000 7ffa57ee8640 -1 rbd::mirror::ImageReplayer: 0x55ccee22e780 [3/1bfda626-752e-40be-8f8c-5733d4c53ac8] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:30.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:30.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:30.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_2 2024-09-18T01:26:30.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_2 snap 2024-09-18T01:26:30.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:30.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:30.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:26:30.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:30.093 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_2@snap 2024-09-18T01:26:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_2 2024-09-18T01:26:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:26:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:30.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:30.768 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_2 2024-09-18T01:26:30.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:30.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:30.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:26:30.769 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-09-18T01:26:30.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-09-18T01:26:30.920 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:26:30.920+0000 7f7748fde640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cecbb701a0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:30.921 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:26:30.920+0000 7f774f7eb640 -1 rbd::mirror::ImageReplayer: 0x55cecb5d9400 [3/0efe7b51-8742-400d-adca-bf1d51dea26b] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:31.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_3 2024-09-18T01:26:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_3 snap 2024-09-18T01:26:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:31.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:26:31.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:31.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_3@snap 2024-09-18T01:26:32.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:32.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_3 2024-09-18T01:26:32.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:32.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:32.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:26:32.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:32.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:32.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_3 2024-09-18T01:26:32.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:32.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:32.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:26:32.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-09-18T01:26:32.826 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-09-18T01:26:32.933 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:26:32.932+0000 7f77487dd640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cecad76820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:32.933 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:26:32.932+0000 7f774efea640 -1 rbd::mirror::ImageReplayer: 0x55cecb5d8780 [3/26ca4ff3-0678-4f67-8c75-7af181831497] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:33.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_4 2024-09-18T01:26:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_4 snap 2024-09-18T01:26:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:26:33.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:33.914 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_4@snap 2024-09-18T01:26:34.778 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:34.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_4 2024-09-18T01:26:34.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:34.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_4 2024-09-18T01:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:34.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:34.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:26:34.791 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-09-18T01:26:34.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-09-18T01:26:34.889 INFO:tasks.ceph.cluster1.osd.1.smithi078.stderr:2024-09-18T01:26:34.887+0000 7f88bb526640 -1 reset not still connected to 0x55b499cf15f0 2024-09-18T01:26:34.889 INFO:tasks.ceph.cluster1.osd.1.smithi078.stderr:2024-09-18T01:26:34.887+0000 7f88bb526640 -1 reset not still connected to 0x55b499cf1790 2024-09-18T01:26:34.951 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:34.950+0000 7ff69d8e3640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2d3169c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:34.951 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:34.950+0000 7ff6a40f0640 -1 rbd::mirror::ImageReplayer: 0x559f2d809680 [3/f06096fd-c27d-4c66-a888-be876f867f5a] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:35.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:35.919 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:35.919 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:35.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_5 2024-09-18T01:26:35.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_5 snap 2024-09-18T01:26:35.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:35.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:35.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:26:35.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:35.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_5@snap 2024-09-18T01:26:36.975 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:36.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_5 2024-09-18T01:26:36.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:36.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:36.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:26:36.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:36.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:36.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_5 2024-09-18T01:26:36.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:36.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:36.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:26:36.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-09-18T01:26:37.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-09-18T01:26:37.138 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:37.138+0000 7ff69e0e4640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2d317380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:37.138 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:37.138+0000 7ff6a40f0640 -1 rbd::mirror::ImageReplayer: 0x559f2d40ea00 [3/abb2637a-7d4a-4f6e-a9ca-ee069dffe536] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:38.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:38.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_6 2024-09-18T01:26:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_6 snap 2024-09-18T01:26:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:26:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:38.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_6@snap 2024-09-18T01:26:38.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_6 2024-09-18T01:26:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:26:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_6 2024-09-18T01:26:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:26:38.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-09-18T01:26:39.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-09-18T01:26:39.154 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:26:39.153+0000 7ffa51edc640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ccedad0b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:39.154 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:26:39.153+0000 7ffa586e9640 -1 rbd::mirror::ImageReplayer: 0x55ccee32a000 [3/cbcb3575-14f3-490d-a7c4-517bdeb6f42a] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:40.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_7 2024-09-18T01:26:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_7 snap 2024-09-18T01:26:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:26:40.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:40.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_7@snap 2024-09-18T01:26:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_7 2024-09-18T01:26:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:26:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:41.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_7 2024-09-18T01:26:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:26:41.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-09-18T01:26:41.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-09-18T01:26:41.154 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:41.153+0000 7ff69e0e4640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2d316d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:42.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_8 2024-09-18T01:26:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_8 snap 2024-09-18T01:26:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:26:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:42.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_8@snap 2024-09-18T01:26:42.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_8 2024-09-18T01:26:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:26:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:43.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:43.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_8 2024-09-18T01:26:43.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:43.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:43.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:26:43.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-09-18T01:26:43.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-09-18T01:26:43.174 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:43.173+0000 7ff69e0e4640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2d317520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:43.174 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:43.173+0000 7ff6a48f1640 -1 rbd::mirror::ImageReplayer: 0x559f2d808780 [3/c8657da4-764c-45a1-9898-eb5d4ab62f4f] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:44.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:44.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:44.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:44.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_9 2024-09-18T01:26:44.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_9 snap 2024-09-18T01:26:44.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:44.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:44.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:26:44.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:44.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_9@snap 2024-09-18T01:26:45.012 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:45.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_9 2024-09-18T01:26:45.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:45.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:45.024 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:26:45.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:45.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:45.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_9 2024-09-18T01:26:45.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:45.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:45.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:26:45.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-09-18T01:26:45.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-09-18T01:26:45.182 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:26:45.181+0000 7f9c198ab640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5647babced00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:45.182 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:26:45.181+0000 7f9c200b8640 -1 rbd::mirror::ImageReplayer: 0x5647bae57900 [3/28c48b31-f556-4f13-90b4-b7bc04a088dd] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:46.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:46.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:46.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:46.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_10 2024-09-18T01:26:46.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_10 snap 2024-09-18T01:26:46.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:46.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:46.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:26:46.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:46.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_10@snap 2024-09-18T01:26:47.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:47.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_10 2024-09-18T01:26:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:26:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:47.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_10 2024-09-18T01:26:47.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:47.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:47.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:26:47.050 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-09-18T01:26:47.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-09-18T01:26:47.204 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:26:47.204+0000 7ffa51edc640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ccedad09c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:47.205 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:26:47.204+0000 7ffa586e9640 -1 rbd::mirror::ImageReplayer: 0x55ccee22e500 [3/0a5efa51-897f-43ff-9dc4-ca67a0c87f5d] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:48.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:48.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_11 2024-09-18T01:26:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_11 snap 2024-09-18T01:26:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:26:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:48.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_11@snap 2024-09-18T01:26:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:49.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_11 2024-09-18T01:26:49.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:49.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:49.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:26:49.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:49.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:49.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_11 2024-09-18T01:26:49.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:49.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:49.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:26:49.055 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_11 2024-09-18T01:26:49.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_11 2024-09-18T01:26:49.206 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:26:49.206+0000 7f77487dd640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cecad776c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:49.206 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:26:49.206+0000 7f774efea640 -1 rbd::mirror::ImageReplayer: 0x55cecb5d8280 [3/067a8b9f-a615-4968-8d24-956a8b2892a6] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:50.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_12 2024-09-18T01:26:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_12 snap 2024-09-18T01:26:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:26:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:50.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_12@snap 2024-09-18T01:26:50.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:51.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_12 2024-09-18T01:26:51.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:51.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:51.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:26:51.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:51.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:51.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_12 2024-09-18T01:26:51.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:51.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:51.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:26:51.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_12 2024-09-18T01:26:51.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_12 2024-09-18T01:26:51.147 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:51.147+0000 7ff69d8e3640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2daf7860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:51.147 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:51.147+0000 7ff6a40f0640 -1 rbd::mirror::ImageReplayer: 0x559f2d808f00 [3/b8fdbcd4-3f49-41c6-90b8-318c1f5ef36b] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:52.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:52.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_13 2024-09-18T01:26:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_13 snap 2024-09-18T01:26:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:26:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_13@snap 2024-09-18T01:26:53.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:53.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_13 2024-09-18T01:26:53.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:53.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:26:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_13 2024-09-18T01:26:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:53.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:53.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:26:53.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_13 2024-09-18T01:26:53.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_13 2024-09-18T01:26:53.230 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:53.229+0000 7ff69e0e4640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2daf7d40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:53.230 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:53.229+0000 7ff6a48f1640 -1 rbd::mirror::ImageReplayer: 0x559f2d40e280 [3/a60dcf1c-e06a-4d99-b676-d35f298225a4] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:54.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_14 2024-09-18T01:26:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_14 snap 2024-09-18T01:26:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:26:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:54.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_14@snap 2024-09-18T01:26:55.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:55.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_14 2024-09-18T01:26:55.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:55.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:55.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:26:55.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:55.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:55.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_14 2024-09-18T01:26:55.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:55.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:55.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:26:55.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_14 2024-09-18T01:26:55.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_14 2024-09-18T01:26:55.239 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:26:55.237+0000 7f9c198ab640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5647babceea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:55.239 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:26:55.237+0000 7f9c200b8640 -1 rbd::mirror::ImageReplayer: 0x5647bae56780 [3/6c6c64a2-50e5-44f2-8703-2da1463fc540] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:56.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_15 2024-09-18T01:26:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_15 snap 2024-09-18T01:26:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:26:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:56.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_15@snap 2024-09-18T01:26:56.632 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6101736072918754) > weight (0.25) 2024-09-18T01:26:56.632 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-09-18T01:26:56.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-09-18T01:26:56.633 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-18T01:26:56.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:26:56.632+0000 7f9c208b9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 49450) UID: 1000 2024-09-18T01:26:57.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_15 2024-09-18T01:26:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:26:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_15 2024-09-18T01:26:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:57.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:26:57.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_15 2024-09-18T01:26:57.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_15 2024-09-18T01:26:57.230 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:57.229+0000 7ff69d8e3640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2ce6f520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:57.230 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:26:57.229+0000 7ff6a48f1640 -1 rbd::mirror::ImageReplayer: 0x559f2d40e780 [3/b04593f0-b419-4bde-acad-9a17209e79b7] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:26:58.211 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:26:58.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:26:58.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:26:58.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_16 2024-09-18T01:26:58.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_16 snap 2024-09-18T01:26:58.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:58.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:58.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:26:58.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:26:58.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_16@snap 2024-09-18T01:26:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:26:59.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_16 2024-09-18T01:26:59.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:59.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:59.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:26:59.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:26:59.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:26:59.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_16 2024-09-18T01:26:59.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:26:59.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:26:59.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:26:59.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_16 2024-09-18T01:26:59.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_16 2024-09-18T01:26:59.248 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:26:59.246+0000 7f77487dd640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cecad76b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:26:59.248 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:26:59.246+0000 7f774efea640 -1 rbd::mirror::ImageReplayer: 0x55cecb5d9680 [3/1c96c904-8436-480e-934e-936299cf345d] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:00.219 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:00.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:00.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:00.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_17 2024-09-18T01:27:00.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_17 snap 2024-09-18T01:27:00.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:00.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:00.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:27:00.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:00.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_17@snap 2024-09-18T01:27:00.633 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-09-18T01:27:00.633 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:27:00.634 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-09-18T01:27:00.634 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-09-18T01:27:00.634 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-09-18T01:27:00.637 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-09-18T01:27:00.637 INFO:tasks.rbd_mirror_thrash:waiting for 79 secs before thrashing 2024-09-18T01:27:00.982 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:00.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_17 2024-09-18T01:27:00.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:00.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:00.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:27:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_17 2024-09-18T01:27:00.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:00.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:00.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:27:00.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_17 2024-09-18T01:27:01.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_17 2024-09-18T01:27:01.146 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:01.144+0000 7f77487dd640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cecae75d40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:01.146 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:01.144+0000 7f774efea640 -1 rbd::mirror::ImageReplayer: 0x55cecb5d8c80 [3/3e9f111f-659a-4c25-a089-ca421c2ebd1a] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:02.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:02.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:02.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:02.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_18 2024-09-18T01:27:02.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_18 snap 2024-09-18T01:27:02.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:02.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:02.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:27:02.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:02.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_18@snap 2024-09-18T01:27:03.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:03.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_18 2024-09-18T01:27:03.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:03.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:03.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:27:03.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:03.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:03.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_18 2024-09-18T01:27:03.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:03.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:03.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:27:03.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_18 2024-09-18T01:27:03.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_18 2024-09-18T01:27:03.251 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:03.250+0000 7ff6a48f1640 -1 rbd::mirror::ImageMap: 0x559f2d824900 handle_peer_ack_remove: failed to remove global_image_id=716a32d0-d4de-4f8b-8c5b-1431e7a8c859 2024-09-18T01:27:04.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:04.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:04.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:04.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_19 2024-09-18T01:27:04.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_19 snap 2024-09-18T01:27:04.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:04.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:04.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:27:04.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:04.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_19@snap 2024-09-18T01:27:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:05.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_19 2024-09-18T01:27:05.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:05.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:05.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:27:05.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:05.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:05.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_19 2024-09-18T01:27:05.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:05.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:05.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:27:05.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_19 2024-09-18T01:27:05.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_19 2024-09-18T01:27:05.266 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:05.264+0000 7ff6a48f1640 -1 rbd::mirror::ImageMap: 0x559f2d824900 handle_peer_ack_remove: failed to remove global_image_id=957171f2-b93c-457d-89be-5d45fdaad96c 2024-09-18T01:27:06.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:06.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:06.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:06.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_20 2024-09-18T01:27:06.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_20 snap 2024-09-18T01:27:06.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:06.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:06.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:27:06.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:06.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_20@snap 2024-09-18T01:27:07.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:07.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_20 2024-09-18T01:27:07.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:07.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:07.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:27:07.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:07.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:07.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_20 2024-09-18T01:27:07.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:07.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:07.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:27:07.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_20 2024-09-18T01:27:07.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_20 2024-09-18T01:27:07.276 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:07.274+0000 7ff69d8e3640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2daf7380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:07.276 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:07.274+0000 7ff6a48f1640 -1 rbd::mirror::ImageReplayer: 0x559f2d40f180 [3/820484e1-ad03-4296-9676-f12697dabdb5] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:08.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_21 2024-09-18T01:27:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_21 snap 2024-09-18T01:27:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:27:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:08.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_21@snap 2024-09-18T01:27:09.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:09.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_21 2024-09-18T01:27:09.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:09.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:09.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:27:09.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:09.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:09.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_21 2024-09-18T01:27:09.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:09.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:09.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:27:09.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_21 2024-09-18T01:27:09.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_21 2024-09-18T01:27:09.281 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:09.280+0000 7f7748fde640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cecb5e8340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:09.281 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:09.280+0000 7f774f7eb640 -1 rbd::mirror::ImageReplayer: 0x55cecb5d8a00 [3/56bbe368-35dd-4230-b5bc-91e7ad099a39] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:09.465 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-18T01:27:09.466 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-18T01:27:09.466 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-18T01:27:09.466 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-18T01:27:09.466 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:27:09.467 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:27:09.465+0000 7f478a2a9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 49257) UID: 1000 2024-09-18T01:27:10.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:10.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:10.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:10.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_22 2024-09-18T01:27:10.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_22 snap 2024-09-18T01:27:10.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:10.248 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:10.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:27:10.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:10.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_22@snap 2024-09-18T01:27:10.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:10.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_22 2024-09-18T01:27:10.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:10.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:10.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:27:10.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:10.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:10.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_22 2024-09-18T01:27:10.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:10.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:10.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:27:10.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_22 2024-09-18T01:27:11.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_22 2024-09-18T01:27:11.148 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:11.146+0000 7ffa516db640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ccedad2d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:11.148 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:11.146+0000 7ffa57ee8640 -1 rbd::mirror::ImageReplayer: 0x55ccee22f400 [3/2b2d5ff2-a08d-4bda-9b74-e9d26e07305c] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:12.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_23 2024-09-18T01:27:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_23 snap 2024-09-18T01:27:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:12.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:27:12.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:12.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_23@snap 2024-09-18T01:27:13.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:13.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_23 2024-09-18T01:27:13.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:13.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:13.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:27:13.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:13.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:13.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_23 2024-09-18T01:27:13.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:13.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:13.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:27:13.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_23 2024-09-18T01:27:13.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_23 2024-09-18T01:27:13.265 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:27:13.260+0000 7f17120ec640 -1 rbd::mirror::ImageReplayer: 0x55e72cf6e280 [3/bf72281c-0895-415a-a775-377773847714] start: already running 2024-09-18T01:27:13.288 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:13.286+0000 7ff69d8e3640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2ce70d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:13.288 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:13.286+0000 7ff6a48f1640 -1 rbd::mirror::ImageReplayer: 0x559f2d808500 [3/bf72281c-0895-415a-a775-377773847714] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:14.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:14.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:14.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:14.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_24 2024-09-18T01:27:14.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_24 snap 2024-09-18T01:27:14.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:14.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:14.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:27:14.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:14.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_24@snap 2024-09-18T01:27:15.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_24 2024-09-18T01:27:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:27:15.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:15.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:15.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_24 2024-09-18T01:27:15.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:15.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:15.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:27:15.157 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_24 2024-09-18T01:27:15.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_24 2024-09-18T01:27:15.309 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:15.308+0000 7ffa51edc640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cced9be680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:15.309 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:15.308+0000 7ffa586e9640 -1 rbd::mirror::ImageReplayer: 0x55ccee32a280 [3/e2f372b0-91b5-4e89-95d5-57ab346b50a4] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:15.568 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-18T01:27:15.568 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-18T01:27:15.569 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-18T01:27:15.571 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-18T01:27:15.572 INFO:tasks.rbd_mirror_thrash:waiting for 84 secs before thrashing 2024-09-18T01:27:16.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:16.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_25 2024-09-18T01:27:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_25 snap 2024-09-18T01:27:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:27:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_25@snap 2024-09-18T01:27:17.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:17.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_25 2024-09-18T01:27:17.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:17.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:17.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:27:17.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:17.161 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:17.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_25 2024-09-18T01:27:17.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:17.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:17.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:27:17.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_25 2024-09-18T01:27:17.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_25 2024-09-18T01:27:17.308 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:17.307+0000 7ffa516db640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cced9be4e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:17.309 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:17.307+0000 7ffa586e9640 -1 rbd::mirror::ImageReplayer: 0x55ccee32aa00 [3/87947b3d-24d4-4600-a788-3246abc4083b] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:18.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:18.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:18.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:18.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_26 2024-09-18T01:27:18.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_26 snap 2024-09-18T01:27:18.287 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:18.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:18.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:27:18.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:18.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_26@snap 2024-09-18T01:27:19.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:19.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_26 2024-09-18T01:27:19.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:19.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:19.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:27:19.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:19.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:19.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_26 2024-09-18T01:27:19.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:19.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:19.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:27:19.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_26 2024-09-18T01:27:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_26 2024-09-18T01:27:19.322 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:19.320+0000 7ff6a48f1640 -1 rbd::mirror::ImageMap: 0x559f2d824900 handle_peer_ack_remove: failed to remove global_image_id=ca67ff60-0889-4b07-bb52-b0e5f8ac56c7 2024-09-18T01:27:20.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_27 2024-09-18T01:27:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_27 snap 2024-09-18T01:27:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:27:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:20.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_27@snap 2024-09-18T01:27:21.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:21.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_27 2024-09-18T01:27:21.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:21.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:21.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:27:21.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:21.010 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_27 2024-09-18T01:27:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:27:21.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_27 2024-09-18T01:27:21.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_27 2024-09-18T01:27:21.157 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:21.157+0000 7ff69d8e3640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2ce70000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:21.157 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:21.157+0000 7ff6a40f0640 -1 rbd::mirror::ImageReplayer: 0x559f2d808c80 [3/d521f001-ba66-4693-9d40-d7a054a004d9] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:22.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:22.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:22.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:22.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_28 2024-09-18T01:27:22.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_28 snap 2024-09-18T01:27:22.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:22.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:27:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:22.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_28@snap 2024-09-18T01:27:23.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_28 2024-09-18T01:27:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:27:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:23.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:23.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_28 2024-09-18T01:27:23.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:23.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:23.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:27:23.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_28 2024-09-18T01:27:23.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_28 2024-09-18T01:27:23.334 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:27:23.334+0000 7f68ca309640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562b53d2a1a0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:23.334 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:27:23.334+0000 7f68d1317640 -1 rbd::mirror::ImageReplayer: 0x562b53d28000 [3/24274cb0-c250-4f16-a3b6-16755405ece7] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:24.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:24.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:24.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:24.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_29 2024-09-18T01:27:24.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_29 snap 2024-09-18T01:27:24.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:24.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:24.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:27:24.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:24.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_29@snap 2024-09-18T01:27:25.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_29 2024-09-18T01:27:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:27:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:25.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:25.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_29 2024-09-18T01:27:25.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:25.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:25.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:27:25.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_29 2024-09-18T01:27:25.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_29 2024-09-18T01:27:25.335 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:25.335+0000 7ff69e0e4640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2cc416c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:25.335 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:25.335+0000 7ff6a40f0640 -1 rbd::mirror::ImageReplayer: 0x559f2d808280 [3/f1b25004-306f-4390-bf36-878a869a0106] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:26.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:26.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:26.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:26.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_30 2024-09-18T01:27:26.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_30 snap 2024-09-18T01:27:26.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:26.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:26.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:27:26.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:26.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_30@snap 2024-09-18T01:27:27.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:27.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_30 2024-09-18T01:27:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:27:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:27.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_30 2024-09-18T01:27:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:27:27.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_30 2024-09-18T01:27:27.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_30 2024-09-18T01:27:28.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:28.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:28.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:28.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_31 2024-09-18T01:27:28.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_31 snap 2024-09-18T01:27:28.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:28.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:28.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:27:28.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:28.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_31@snap 2024-09-18T01:27:29.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:29.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_31 2024-09-18T01:27:29.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:29.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:29.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:27:29.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:29.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:29.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_31 2024-09-18T01:27:29.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:29.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:29.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:27:29.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_31 2024-09-18T01:27:29.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_31 2024-09-18T01:27:30.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:30.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:30.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:30.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_32 2024-09-18T01:27:30.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_32 snap 2024-09-18T01:27:30.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:30.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:30.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:27:30.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:30.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_32@snap 2024-09-18T01:27:30.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:31.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_32 2024-09-18T01:27:31.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:31.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:31.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:27:31.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:31.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:31.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_32 2024-09-18T01:27:31.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:31.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:31.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:27:31.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_32 2024-09-18T01:27:31.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_32 2024-09-18T01:27:31.155 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:31.154+0000 7ffa516db640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ccee2204e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:31.155 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:31.154+0000 7ffa57ee8640 -1 rbd::mirror::ImageReplayer: 0x55ccee22fb80 [3/884d933d-d88c-4657-97ca-5aa2e3991921] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:32.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_33 2024-09-18T01:27:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_33 snap 2024-09-18T01:27:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:27:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:32.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_33@snap 2024-09-18T01:27:33.081 INFO:tasks.ceph.cluster1.osd.0.smithi078.stderr:2024-09-18T01:27:33.080+0000 7f7bd2549640 -1 reset not still connected to 0x561e51244c30 2024-09-18T01:27:33.081 INFO:tasks.ceph.cluster1.osd.0.smithi078.stderr:2024-09-18T01:27:33.080+0000 7f7bd2549640 -1 reset not still connected to 0x561e51fd5d40 2024-09-18T01:27:33.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:33.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_33 2024-09-18T01:27:33.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:33.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:33.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:27:33.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:33.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:33.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_33 2024-09-18T01:27:33.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:33.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:33.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:27:33.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_33 2024-09-18T01:27:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_33 2024-09-18T01:27:33.540 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:33.540+0000 7f7748fde640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cecad769c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:33.540 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:33.540+0000 7f774f7eb640 -1 rbd::mirror::ImageReplayer: 0x55cecb5d8f00 [3/4d961941-4020-4df3-821f-ef25e429389f] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:34.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:34.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_34 2024-09-18T01:27:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_34 snap 2024-09-18T01:27:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:27:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:34.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_34@snap 2024-09-18T01:27:35.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:35.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_34 2024-09-18T01:27:35.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:35.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:35.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:27:35.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:35.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:35.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_34 2024-09-18T01:27:35.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:35.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:35.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:27:35.200 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_34 2024-09-18T01:27:35.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_34 2024-09-18T01:27:35.356 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:35.356+0000 7f7748fde640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cecae75ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:35.356 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:35.356+0000 7f774efea640 -1 rbd::mirror::ImageReplayer: 0x55cecb6c2000 [3/4770a819-7931-426e-a8e9-0d6c6ec6d2de] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:36.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:36.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:36.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:36.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_35 2024-09-18T01:27:36.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_35 snap 2024-09-18T01:27:36.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:36.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:36.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:27:36.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:36.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_35@snap 2024-09-18T01:27:36.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:36.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_35 2024-09-18T01:27:36.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:36.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:36.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:27:36.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:36.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:36.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_35 2024-09-18T01:27:36.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:36.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:36.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:27:36.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_35 2024-09-18T01:27:37.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_35 2024-09-18T01:27:37.157 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:37.156+0000 7f77487dd640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cecae75860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:37.157 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:37.156+0000 7f774efea640 -1 rbd::mirror::ImageReplayer: 0x55cecb6c2280 [3/855097c3-5440-41cd-a767-29e218b7cf40] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:38.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:38.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:38.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:38.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_36 2024-09-18T01:27:38.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_36 snap 2024-09-18T01:27:38.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:38.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:38.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:27:38.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:38.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_36@snap 2024-09-18T01:27:38.995 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:39.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_36 2024-09-18T01:27:39.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:39.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:39.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:27:39.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:39.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_36 2024-09-18T01:27:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:27:39.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_36 2024-09-18T01:27:39.055 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_36 2024-09-18T01:27:39.165 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:39.165+0000 7f77487dd640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cecae74d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:39.165 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:27:39.165+0000 7f774efea640 -1 rbd::mirror::ImageReplayer: 0x55cecb5d9b80 [3/3ee14da1-57b2-4d60-bd27-8af0515acf51] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:40.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_37 2024-09-18T01:27:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_37 snap 2024-09-18T01:27:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:27:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:40.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_37@snap 2024-09-18T01:27:40.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_37 2024-09-18T01:27:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:27:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:41.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_37 2024-09-18T01:27:41.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:27:41.002 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_37 2024-09-18T01:27:41.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_37 2024-09-18T01:27:42.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:42.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:42.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:42.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_38 2024-09-18T01:27:42.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_38 snap 2024-09-18T01:27:42.140 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:42.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:42.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:27:42.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:42.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_38@snap 2024-09-18T01:27:43.005 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:43.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_38 2024-09-18T01:27:43.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:43.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:43.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:27:43.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:43.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:43.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_38 2024-09-18T01:27:43.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:43.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:43.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:27:43.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_38 2024-09-18T01:27:43.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_38 2024-09-18T01:27:43.174 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:27:43.173+0000 7f68cab0a640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562b534a7040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:43.174 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:27:43.174+0000 7f68d0b16640 -1 rbd::mirror::ImageReplayer: 0x562b53d28c80 [3/4ff798bb-162f-4ad6-acae-69c5a62867d9] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:44.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:44.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:44.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:44.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_39 2024-09-18T01:27:44.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_39 snap 2024-09-18T01:27:44.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:44.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:44.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:27:44.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:44.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_39@snap 2024-09-18T01:27:45.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:45.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_39 2024-09-18T01:27:45.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:45.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:45.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:27:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_39 2024-09-18T01:27:45.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:45.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:45.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:27:45.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_39 2024-09-18T01:27:45.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_39 2024-09-18T01:27:45.181 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:45.180+0000 7ffa51edc640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ccee220820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:45.182 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:45.180+0000 7ffa57ee8640 -1 rbd::mirror::ImageReplayer: 0x55ccee22f900 [3/6f579a7e-81b1-4594-9e4f-741a03c86452] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:46.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_40 2024-09-18T01:27:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_40 snap 2024-09-18T01:27:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:27:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:46.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_40@snap 2024-09-18T01:27:47.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:47.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_40 2024-09-18T01:27:47.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:47.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:47.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:27:47.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:47.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:47.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_40 2024-09-18T01:27:47.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:47.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:47.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:27:47.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_40 2024-09-18T01:27:47.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_40 2024-09-18T01:27:47.207 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:47.205+0000 7ffa516db640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cced9bf380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:47.207 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:47.205+0000 7ffa586e9640 -1 rbd::mirror::ImageReplayer: 0x55ccee22f680 [3/75569094-ed91-4db4-95b0-2512a6bdcc51] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:48.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_41 2024-09-18T01:27:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_41 snap 2024-09-18T01:27:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:27:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:48.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_41@snap 2024-09-18T01:27:49.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_41 2024-09-18T01:27:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:49.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:49.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:27:49.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:49.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_41 2024-09-18T01:27:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:27:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_41 2024-09-18T01:27:49.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_41 2024-09-18T01:27:49.202 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:49.201+0000 7ffa516db640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cced9bb860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:49.203 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:49.201+0000 7ffa586e9640 -1 rbd::mirror::ImageReplayer: 0x55ccee22e280 [3/0071b191-d397-4a11-b33d-d10f4477a5a3] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:50.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_42 2024-09-18T01:27:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_42 snap 2024-09-18T01:27:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:27:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:50.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_42@snap 2024-09-18T01:27:51.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_42 2024-09-18T01:27:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:27:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:51.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:51.017 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_42 2024-09-18T01:27:51.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:51.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:51.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:27:51.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_42 2024-09-18T01:27:51.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_42 2024-09-18T01:27:51.171 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:27:51.170+0000 7f68ca309640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562b53d2a340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:51.171 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:27:51.170+0000 7f68d1317640 -1 rbd::mirror::ImageReplayer: 0x562b53d28780 [3/aeaeb058-01f1-4690-999c-8c21dad1561e] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:52.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:52.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:52.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:52.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_43 2024-09-18T01:27:52.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_43 snap 2024-09-18T01:27:52.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:52.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:52.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:27:52.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:52.184 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_43@snap 2024-09-18T01:27:53.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:53.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_43 2024-09-18T01:27:53.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:53.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:53.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:27:53.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:53.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:53.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_43 2024-09-18T01:27:53.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:53.059 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:27:53.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_43 2024-09-18T01:27:53.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_43 2024-09-18T01:27:53.233 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:53.231+0000 7ff69e0e4640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2d3169c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:53.233 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:53.231+0000 7ff6a40f0640 -1 rbd::mirror::ImageReplayer: 0x559f2d40e500 [3/bf14a2b6-1625-4fba-8a13-3e69c7467ad0] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:54.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:54.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:54.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:54.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_44 2024-09-18T01:27:54.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_44 snap 2024-09-18T01:27:54.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:54.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:54.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:27:54.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:54.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_44@snap 2024-09-18T01:27:55.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:55.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_44 2024-09-18T01:27:55.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:55.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:55.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:27:55.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:55.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:55.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_44 2024-09-18T01:27:55.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:55.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:55.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:27:55.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_44 2024-09-18T01:27:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_44 2024-09-18T01:27:55.236 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:55.234+0000 7ff69e0e4640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2d317380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:55.236 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:27:55.234+0000 7ff6a48f1640 -1 rbd::mirror::ImageReplayer: 0x559f2d40e280 [3/f0afae9f-39f5-4655-a91e-7d6dfba2149c] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:56.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_45 2024-09-18T01:27:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_45 snap 2024-09-18T01:27:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:27:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:56.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_45@snap 2024-09-18T01:27:57.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_45 2024-09-18T01:27:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:27:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:57.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_45 2024-09-18T01:27:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:27:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_45 2024-09-18T01:27:57.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_45 2024-09-18T01:27:57.238 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:57.237+0000 7ffa516db640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ccee220ea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:57.239 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:27:57.237+0000 7ffa57ee8640 -1 rbd::mirror::ImageReplayer: 0x55ccee22ec80 [3/4858f75f-8426-4450-aea4-9c743ab91e23] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:27:58.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:27:58.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:27:58.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:27:58.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_46 2024-09-18T01:27:58.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_46 snap 2024-09-18T01:27:58.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:58.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:58.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:27:58.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:27:58.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_46@snap 2024-09-18T01:27:59.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:27:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_46 2024-09-18T01:27:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:59.087 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:59.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:27:59.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:27:59.088 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:27:59.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_46 2024-09-18T01:27:59.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:27:59.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:27:59.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:27:59.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_46 2024-09-18T01:27:59.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_46 2024-09-18T01:27:59.260 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:27:59.259+0000 7f68ca309640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562b537c5040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:27:59.261 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:27:59.259+0000 7f68d1317640 -1 rbd::mirror::ImageReplayer: 0x562b53d28f00 [3/48442b3f-58cc-4f8f-8a64-4a412ad18406] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:28:00.210 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:28:00.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:00.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:00.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_47 2024-09-18T01:28:00.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_47 snap 2024-09-18T01:28:00.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:00.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:00.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:28:00.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:28:00.214 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_47@snap 2024-09-18T01:28:01.006 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:28:01.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_47 2024-09-18T01:28:01.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:01.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:01.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:28:01.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:28:01.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:28:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_47 2024-09-18T01:28:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:28:01.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_47 2024-09-18T01:28:01.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_47 2024-09-18T01:28:01.170 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:28:01.169+0000 7f7748fde640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cecb369520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:28:01.170 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:28:01.169+0000 7f774efea640 -1 rbd::mirror::ImageReplayer: 0x55cecb6c2500 [3/dc5f334a-10d4-4c3c-9345-0b305199e1f2] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:28:02.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:28:02.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:02.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:02.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_48 2024-09-18T01:28:02.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_48 snap 2024-09-18T01:28:02.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:02.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:02.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:28:02.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:28:02.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_48@snap 2024-09-18T01:28:03.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:28:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_48 2024-09-18T01:28:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:03.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:28:03.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:28:03.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:28:03.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_48 2024-09-18T01:28:03.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:03.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:03.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:28:03.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_48 2024-09-18T01:28:03.151 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_48 2024-09-18T01:28:03.254 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:28:03.252+0000 7f68cab0a640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562b537c4ea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:28:03.254 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:28:03.252+0000 7f68d0b16640 -1 rbd::mirror::ImageReplayer: 0x562b53d28a00 [3/db673391-3699-4385-93be-31f993e37bad] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:28:04.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:28:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_49 2024-09-18T01:28:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_49 snap 2024-09-18T01:28:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:28:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:28:04.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_49@snap 2024-09-18T01:28:05.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:28:05.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_49 2024-09-18T01:28:05.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:05.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:05.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:28:05.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:28:05.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:28:05.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_49 2024-09-18T01:28:05.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:05.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:05.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:28:05.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_49 2024-09-18T01:28:05.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_49 2024-09-18T01:28:05.265 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:28:05.264+0000 7ffa51edc640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cced9be4e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:28:05.265 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:28:05.264+0000 7ffa57ee8640 -1 rbd::mirror::ImageReplayer: 0x55ccee22ea00 [3/380bbc6e-0c06-47e3-a103-ff2ba6ff04d9] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:28:06.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:28:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_50 2024-09-18T01:28:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_snapshot cluster2 mirror image_50 snap 2024-09-18T01:28:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:28:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local snap=snap 2024-09-18T01:28:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap rm mirror/image_50@snap 2024-09-18T01:28:07.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing snap: 100% complete...done. 2024-09-18T01:28:07.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_50 2024-09-18T01:28:07.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:07.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:07.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:28:07.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:28:07.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:28:07.108 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_50 2024-09-18T01:28:07.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:07.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:07.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:28:07.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_50 2024-09-18T01:28:07.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_50 2024-09-18T01:28:07.272 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:28:07.271+0000 7ff69e0e4640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x559f2d316d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:28:07.273 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:28:07.271+0000 7ff6a48f1640 -1 rbd::mirror::ImageReplayer: 0x559f2d808a00 [3/e589d353-55de-4143-8072-8a9ace421dbf] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:28:08.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-09-18T01:28:08.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ testlog 'TEST: image deletions should propagate' 2024-09-18T01:28:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ tee -a /tmp/tmp.yeNdCEAq6i/rbd-mirror.test.log 2024-09-18T01:28:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ date '+%F %T' 2024-09-18T01:28:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo 2024-09-18 01:28:08 TEST: image deletions should propagate 2024-09-18T01:28:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:2024-09-18 01:28:08 TEST: image deletions should propagate 2024-09-18T01:28:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_pool_images cluster1 mirror 0 2024-09-18T01:28:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_count=0 2024-09-18T01:28:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local count 2024-09-18T01:28:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local last_count=0 2024-09-18T01:28:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ true 2024-09-18T01:28:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 40 2024-09-18T01:28:08.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in `seq 1 40` 2024-09-18T01:28:08.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 1 -ne 1 2024-09-18T01:28:08.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-18T01:28:08.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep 'images: ' 2024-09-18T01:28:08.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cut '-d ' -f 2 2024-09-18T01:28:08.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: failed to query services: (13) Permission denied 2024-09-18T01:28:08.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ count=0 2024-09-18T01:28:08.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 0 = 0 2024-09-18T01:28:08.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:08.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_pool_healthy cluster1 mirror 0 2024-09-18T01:28:08.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:08.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:08.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:08.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state 2024-09-18T01:28:08.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 40 2024-09-18T01:28:08.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in `seq 1 40` 2024-09-18T01:28:08.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 1 -ne 1 2024-09-18T01:28:08.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-09-18T01:28:08.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep 'image health:' 2024-09-18T01:28:08.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cut '-d ' -f 3 2024-09-18T01:28:08.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: failed to query services: (13) Permission denied 2024-09-18T01:28:08.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ state=OK 2024-09-18T01:28:08.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test OK = ERROR 2024-09-18T01:28:08.320 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test OK = OK 2024-09-18T01:28:08.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:08.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 50 2024-09-18T01:28:08.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:08.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_1 2024-09-18T01:28:08.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2024-09-18T01:28:08.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:08.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:08.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:08.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:08.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:08.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:08.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:08.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_1 2024-09-18T01:28:08.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:08.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:08.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_1 2024-09-18T01:28:08.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-18T01:28:08.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:08.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-09-18T01:28:08.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:08.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:08.358 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2024-09-18T01:28:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:08.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:08.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:08.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:08.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:08.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:08.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_1 2024-09-18T01:28:08.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:08.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:08.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_1 2024-09-18T01:28:08.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-18T01:28:08.460 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:08.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-09-18T01:28:08.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_2 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:08.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:08.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_2 2024-09-18T01:28:08.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:08.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:08.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_2 2024-09-18T01:28:08.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-18T01:28:08.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:08.528 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-18T01:28:08.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:08.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:08.531 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2024-09-18T01:28:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:28:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_2 2024-09-18T01:28:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_2 2024-09-18T01:28:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-18T01:28:08.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:08.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-18T01:28:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:08.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_3 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_3 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:08.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:08.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_3 2024-09-18T01:28:08.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-18T01:28:08.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:08.704 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-09-18T01:28:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:08.707 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:08.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2024-09-18T01:28:08.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:08.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:08.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:28:08.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:08.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:08.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:08.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:08.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_3 2024-09-18T01:28:08.809 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:08.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:08.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_3 2024-09-18T01:28:08.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-18T01:28:08.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:08.843 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-09-18T01:28:08.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:08.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:08.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:08.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:08.847 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_4 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_4 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:08.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_4 2024-09-18T01:28:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-18T01:28:08.849 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:08.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-09-18T01:28:08.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:08.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:08.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:08.989 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2024-09-18T01:28:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:28:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_4 2024-09-18T01:28:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:08.990 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_4 2024-09-18T01:28:08.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-18T01:28:08.991 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.023 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-09-18T01:28:09.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:09.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:09.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_5 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_5 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:09.028 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:09.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_5 2024-09-18T01:28:09.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-18T01:28:09.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-18T01:28:09.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:09.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:09.063 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2024-09-18T01:28:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:28:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_5 2024-09-18T01:28:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:09.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_5 2024-09-18T01:28:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-18T01:28:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-18T01:28:09.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:09.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_6 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.203 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_6 2024-09-18T01:28:09.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:09.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:09.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_6 2024-09-18T01:28:09.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-18T01:28:09.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-09-18T01:28:09.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:09.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:09.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2024-09-18T01:28:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:28:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:09.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_6 2024-09-18T01:28:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_6 2024-09-18T01:28:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-18T01:28:09.341 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-09-18T01:28:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:09.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_7 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_7 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:09.379 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_7 2024-09-18T01:28:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-18T01:28:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.412 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-18T01:28:09.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:09.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:09.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:09.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2024-09-18T01:28:09.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:09.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:09.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:28:09.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:09.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:09.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:09.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:09.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_7 2024-09-18T01:28:09.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:09.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:09.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_7 2024-09-18T01:28:09.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-18T01:28:09.518 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-18T01:28:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_8 2024-09-18T01:28:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2024-09-18T01:28:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:09.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:28:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_8 2024-09-18T01:28:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_8 2024-09-18T01:28:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-18T01:28:09.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.587 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-18T01:28:09.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:09.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:09.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2024-09-18T01:28:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:28:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:09.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_8 2024-09-18T01:28:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_8 2024-09-18T01:28:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-18T01:28:09.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.726 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-18T01:28:09.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:09.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:09.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_9 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_9 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:09.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_9 2024-09-18T01:28:09.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-18T01:28:09.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.763 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-18T01:28:09.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:09.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:09.766 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:09.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2024-09-18T01:28:09.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:09.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:09.868 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:28:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_9 2024-09-18T01:28:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_9 2024-09-18T01:28:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-18T01:28:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-18T01:28:09.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:09.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:09.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:09.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_10 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_10 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:09.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_10 2024-09-18T01:28:09.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-18T01:28:09.911 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:09.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-18T01:28:09.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:10.047 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_10 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_10 2024-09-18T01:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-18T01:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-18T01:28:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_11 2024-09-18T01:28:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_11 deleted 2024-09-18T01:28:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:28:10.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:10.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:10.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_11 2024-09-18T01:28:10.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_11 2024-09-18T01:28:10.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2024-09-18T01:28:10.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.117 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_11: (2) No such file or directory 2024-09-18T01:28:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:10.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:10.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:10.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_11 deleted '' 2024-09-18T01:28:10.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_11 2024-09-18T01:28:10.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:10.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:10.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:10.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_11 2024-09-18T01:28:10.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_11 2024-09-18T01:28:10.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2024-09-18T01:28:10.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.255 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_11: (2) No such file or directory 2024-09-18T01:28:10.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:10.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_12 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_12 deleted 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_12 2024-09-18T01:28:10.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_12 2024-09-18T01:28:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2024-09-18T01:28:10.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_12: (2) No such file or directory 2024-09-18T01:28:10.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:10.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:10.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:10.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_12 deleted '' 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_12 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_12 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_12 2024-09-18T01:28:10.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2024-09-18T01:28:10.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.431 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_12: (2) No such file or directory 2024-09-18T01:28:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_13 2024-09-18T01:28:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_13 deleted 2024-09-18T01:28:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:28:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_13 2024-09-18T01:28:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_13 2024-09-18T01:28:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2024-09-18T01:28:10.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_13: (2) No such file or directory 2024-09-18T01:28:10.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:10.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:10.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:10.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_13 deleted '' 2024-09-18T01:28:10.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_13 2024-09-18T01:28:10.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:10.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:10.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:10.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_13 2024-09-18T01:28:10.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_13 2024-09-18T01:28:10.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2024-09-18T01:28:10.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.606 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_13: (2) No such file or directory 2024-09-18T01:28:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_14 2024-09-18T01:28:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_14 deleted 2024-09-18T01:28:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:28:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_14 2024-09-18T01:28:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_14 2024-09-18T01:28:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2024-09-18T01:28:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.642 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_14: (2) No such file or directory 2024-09-18T01:28:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:10.645 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:10.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_14 deleted '' 2024-09-18T01:28:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_14 2024-09-18T01:28:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_14 2024-09-18T01:28:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_14 2024-09-18T01:28:10.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2024-09-18T01:28:10.748 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.780 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_14: (2) No such file or directory 2024-09-18T01:28:10.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:10.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_15 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_15 deleted 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_15 2024-09-18T01:28:10.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_15 2024-09-18T01:28:10.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2024-09-18T01:28:10.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.817 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_15: (2) No such file or directory 2024-09-18T01:28:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:10.820 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_15 deleted '' 2024-09-18T01:28:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_15 2024-09-18T01:28:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:10.922 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:10.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:10.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_15 2024-09-18T01:28:10.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_15 2024-09-18T01:28:10.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2024-09-18T01:28:10.923 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.955 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_15: (2) No such file or directory 2024-09-18T01:28:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_16 2024-09-18T01:28:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_16 deleted 2024-09-18T01:28:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:10.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:28:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_16 2024-09-18T01:28:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_16 2024-09-18T01:28:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2024-09-18T01:28:10.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:10.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_16: (2) No such file or directory 2024-09-18T01:28:10.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:10.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:10.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:11.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_16 deleted '' 2024-09-18T01:28:11.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:11.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:11.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_16 2024-09-18T01:28:11.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_16 2024-09-18T01:28:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_16 2024-09-18T01:28:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2024-09-18T01:28:11.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:11.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_16: (2) No such file or directory 2024-09-18T01:28:11.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:11.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:11.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:11.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:11.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:11.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_17 2024-09-18T01:28:11.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_17 deleted 2024-09-18T01:28:11.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_17 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_17 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2024-09-18T01:28:11.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:11.168 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_17: (2) No such file or directory 2024-09-18T01:28:11.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:11.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:11.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_17 deleted '' 2024-09-18T01:28:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_17 2024-09-18T01:28:11.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_17 2024-09-18T01:28:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_17 2024-09-18T01:28:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2024-09-18T01:28:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:11.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_17: (2) No such file or directory 2024-09-18T01:28:11.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_18 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_18 deleted 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:11.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_18 2024-09-18T01:28:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_18 2024-09-18T01:28:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2024-09-18T01:28:11.312 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:11.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_18: (2) No such file or directory 2024-09-18T01:28:11.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:11.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:11.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_18 deleted '' 2024-09-18T01:28:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_18 2024-09-18T01:28:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_18 2024-09-18T01:28:11.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_18 2024-09-18T01:28:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2024-09-18T01:28:11.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:11.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_18: (2) No such file or directory 2024-09-18T01:28:11.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:11.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:11.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:11.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:11.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:11.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_19 2024-09-18T01:28:11.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_19 deleted 2024-09-18T01:28:11.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:11.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:28:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_19 2024-09-18T01:28:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_19 2024-09-18T01:28:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2024-09-18T01:28:11.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:11.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_19: (2) No such file or directory 2024-09-18T01:28:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:11.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:11.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:11.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_19 deleted '' 2024-09-18T01:28:11.625 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_19 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_19 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_19 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2024-09-18T01:28:11.626 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:11.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_19: (2) No such file or directory 2024-09-18T01:28:11.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:11.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:11.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:11.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:11.663 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_20 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_20 deleted 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_20 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_20 2024-09-18T01:28:11.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2024-09-18T01:28:11.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:11.696 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_20: (2) No such file or directory 2024-09-18T01:28:11.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:11.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:11.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_20 deleted '' 2024-09-18T01:28:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:11.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_20 2024-09-18T01:28:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_20 2024-09-18T01:28:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_20 2024-09-18T01:28:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2024-09-18T01:28:11.802 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:11.835 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_20: (2) No such file or directory 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_21 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_21 deleted 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:11.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:11.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:11.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_21 2024-09-18T01:28:11.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:11.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:11.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_21 2024-09-18T01:28:11.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2024-09-18T01:28:11.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:11.873 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_21: (2) No such file or directory 2024-09-18T01:28:11.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:11.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:11.876 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_21 deleted '' 2024-09-18T01:28:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_21 2024-09-18T01:28:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_21 2024-09-18T01:28:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_21 2024-09-18T01:28:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2024-09-18T01:28:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.011 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_21: (2) No such file or directory 2024-09-18T01:28:12.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:12.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:12.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:12.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:12.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_22 2024-09-18T01:28:12.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_22 deleted 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_22 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_22 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2024-09-18T01:28:12.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.048 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_22: (2) No such file or directory 2024-09-18T01:28:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:12.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:12.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_22 deleted '' 2024-09-18T01:28:12.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:12.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:12.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_22 2024-09-18T01:28:12.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:12.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:12.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:12.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:12.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_22 2024-09-18T01:28:12.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:12.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:12.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_22 2024-09-18T01:28:12.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2024-09-18T01:28:12.154 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_22: (2) No such file or directory 2024-09-18T01:28:12.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:12.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:12.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:12.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:12.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_23 2024-09-18T01:28:12.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_23 deleted 2024-09-18T01:28:12.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:12.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:28:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_23 2024-09-18T01:28:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_23 2024-09-18T01:28:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2024-09-18T01:28:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.225 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_23: (2) No such file or directory 2024-09-18T01:28:12.227 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:12.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:12.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:12.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_23 deleted '' 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_23 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_23 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_23 2024-09-18T01:28:12.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2024-09-18T01:28:12.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.362 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_23: (2) No such file or directory 2024-09-18T01:28:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_24 2024-09-18T01:28:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_24 deleted 2024-09-18T01:28:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:28:12.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_24 2024-09-18T01:28:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_24 2024-09-18T01:28:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2024-09-18T01:28:12.368 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_24: (2) No such file or directory 2024-09-18T01:28:12.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:12.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:12.402 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_24 deleted '' 2024-09-18T01:28:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_24 2024-09-18T01:28:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_24 2024-09-18T01:28:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:12.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_24 2024-09-18T01:28:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2024-09-18T01:28:12.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_24: (2) No such file or directory 2024-09-18T01:28:12.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:12.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_25 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_25 deleted 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_25 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:12.542 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:12.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_25 2024-09-18T01:28:12.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2024-09-18T01:28:12.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_25: (2) No such file or directory 2024-09-18T01:28:12.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:12.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:12.578 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:12.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_25 deleted '' 2024-09-18T01:28:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_25 2024-09-18T01:28:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_25 2024-09-18T01:28:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:12.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:12.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_25 2024-09-18T01:28:12.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2024-09-18T01:28:12.681 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.713 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_25: (2) No such file or directory 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_26 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_26 deleted 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:12.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:12.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:12.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_26 2024-09-18T01:28:12.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:12.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:12.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_26 2024-09-18T01:28:12.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2024-09-18T01:28:12.719 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_26: (2) No such file or directory 2024-09-18T01:28:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:12.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_26 deleted '' 2024-09-18T01:28:12.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:12.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:12.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_26 2024-09-18T01:28:12.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:12.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:12.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:12.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:12.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_26 2024-09-18T01:28:12.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:12.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:12.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_26 2024-09-18T01:28:12.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2024-09-18T01:28:12.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.888 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_26: (2) No such file or directory 2024-09-18T01:28:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_27 2024-09-18T01:28:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_27 deleted 2024-09-18T01:28:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:28:12.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:12.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:12.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:12.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:12.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_27 2024-09-18T01:28:12.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:12.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:12.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_27 2024-09-18T01:28:12.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2024-09-18T01:28:12.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:12.926 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_27: (2) No such file or directory 2024-09-18T01:28:12.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:12.929 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:12.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:13.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_27 deleted '' 2024-09-18T01:28:13.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_27 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_27 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_27 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2024-09-18T01:28:13.032 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_27: (2) No such file or directory 2024-09-18T01:28:13.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:13.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:13.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:13.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:13.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_28 2024-09-18T01:28:13.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_28 deleted 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_28 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_28 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2024-09-18T01:28:13.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_28: (2) No such file or directory 2024-09-18T01:28:13.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:13.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:13.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:13.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_28 deleted '' 2024-09-18T01:28:13.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_28 2024-09-18T01:28:13.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:13.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:13.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:13.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_28 2024-09-18T01:28:13.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_28 2024-09-18T01:28:13.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2024-09-18T01:28:13.208 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.240 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_28: (2) No such file or directory 2024-09-18T01:28:13.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:13.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:13.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:13.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:13.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_29 2024-09-18T01:28:13.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_29 deleted 2024-09-18T01:28:13.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_29 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_29 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2024-09-18T01:28:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_29: (2) No such file or directory 2024-09-18T01:28:13.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:13.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:13.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_29 deleted '' 2024-09-18T01:28:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_29 2024-09-18T01:28:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_29 2024-09-18T01:28:13.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_29 2024-09-18T01:28:13.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2024-09-18T01:28:13.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_29: (2) No such file or directory 2024-09-18T01:28:13.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:13.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:13.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:13.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:13.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_30 2024-09-18T01:28:13.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_30 deleted 2024-09-18T01:28:13.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:28:13.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:13.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:13.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_30 2024-09-18T01:28:13.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_30 2024-09-18T01:28:13.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2024-09-18T01:28:13.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_30: (2) No such file or directory 2024-09-18T01:28:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:13.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:13.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_30 deleted '' 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_30 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_30 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_30 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2024-09-18T01:28:13.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_30: (2) No such file or directory 2024-09-18T01:28:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:13.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_31 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_31 deleted 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_31 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_31 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2024-09-18T01:28:13.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_31: (2) No such file or directory 2024-09-18T01:28:13.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:13.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:13.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:13.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_31 deleted '' 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_31 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_31 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_31 2024-09-18T01:28:13.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2024-09-18T01:28:13.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_31: (2) No such file or directory 2024-09-18T01:28:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_32 2024-09-18T01:28:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_32 deleted 2024-09-18T01:28:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:28:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_32 2024-09-18T01:28:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_32 2024-09-18T01:28:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2024-09-18T01:28:13.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_32: (2) No such file or directory 2024-09-18T01:28:13.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:13.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:13.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:13.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_32 deleted '' 2024-09-18T01:28:13.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_32 2024-09-18T01:28:13.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:13.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:13.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:13.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_32 2024-09-18T01:28:13.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_32 2024-09-18T01:28:13.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2024-09-18T01:28:13.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.943 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_32: (2) No such file or directory 2024-09-18T01:28:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_33 2024-09-18T01:28:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_33 deleted 2024-09-18T01:28:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:13.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:28:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_33 2024-09-18T01:28:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_33 2024-09-18T01:28:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2024-09-18T01:28:13.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:13.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_33: (2) No such file or directory 2024-09-18T01:28:13.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:13.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:13.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:14.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_33 deleted '' 2024-09-18T01:28:14.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_33 2024-09-18T01:28:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_33 2024-09-18T01:28:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_33 2024-09-18T01:28:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2024-09-18T01:28:14.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:14.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_33: (2) No such file or directory 2024-09-18T01:28:14.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:14.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_34 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_34 deleted 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_34 2024-09-18T01:28:14.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_34 2024-09-18T01:28:14.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2024-09-18T01:28:14.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:14.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_34: (2) No such file or directory 2024-09-18T01:28:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_34 deleted '' 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_34 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_34 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_34 2024-09-18T01:28:14.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2024-09-18T01:28:14.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:14.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_34: (2) No such file or directory 2024-09-18T01:28:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_35 2024-09-18T01:28:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_35 deleted 2024-09-18T01:28:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:28:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_35 2024-09-18T01:28:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_35 2024-09-18T01:28:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2024-09-18T01:28:14.298 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:14.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_35: (2) No such file or directory 2024-09-18T01:28:14.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:14.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:14.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:14.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_35 deleted '' 2024-09-18T01:28:14.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_35 2024-09-18T01:28:14.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:14.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:14.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:14.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_35 2024-09-18T01:28:14.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.435 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_35 2024-09-18T01:28:14.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2024-09-18T01:28:14.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:14.468 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_35: (2) No such file or directory 2024-09-18T01:28:14.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:14.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:14.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_36 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_36 deleted 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_36 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.473 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_36 2024-09-18T01:28:14.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2024-09-18T01:28:14.474 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:14.505 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_36: (2) No such file or directory 2024-09-18T01:28:14.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:14.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:14.508 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_36 deleted '' 2024-09-18T01:28:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_36 2024-09-18T01:28:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:14.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_36 2024-09-18T01:28:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_36 2024-09-18T01:28:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2024-09-18T01:28:14.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:14.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_36: (2) No such file or directory 2024-09-18T01:28:14.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_37 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_37 deleted 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_37 2024-09-18T01:28:14.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_37 2024-09-18T01:28:14.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2024-09-18T01:28:14.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:14.680 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_37: (2) No such file or directory 2024-09-18T01:28:14.683 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:14.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:14.684 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_37 deleted '' 2024-09-18T01:28:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_37 2024-09-18T01:28:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_37 2024-09-18T01:28:14.786 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_37 2024-09-18T01:28:14.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2024-09-18T01:28:14.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:14.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_37: (2) No such file or directory 2024-09-18T01:28:14.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:14.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:14.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_38 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_38 deleted 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_38 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_38 2024-09-18T01:28:14.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2024-09-18T01:28:14.825 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:14.855 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_38: (2) No such file or directory 2024-09-18T01:28:14.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:14.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:14.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:14.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_38 deleted '' 2024-09-18T01:28:14.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_38 2024-09-18T01:28:14.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:14.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_38 2024-09-18T01:28:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_38 2024-09-18T01:28:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2024-09-18T01:28:14.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:14.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_38: (2) No such file or directory 2024-09-18T01:28:14.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_39 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_39 deleted 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:14.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:14.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_39 2024-09-18T01:28:14.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:14.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:14.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_39 2024-09-18T01:28:14.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2024-09-18T01:28:14.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:15.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_39: (2) No such file or directory 2024-09-18T01:28:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:15.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_39 deleted '' 2024-09-18T01:28:15.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:15.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:15.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_39 2024-09-18T01:28:15.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:15.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:15.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:15.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:15.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_39 2024-09-18T01:28:15.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:15.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:15.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_39 2024-09-18T01:28:15.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2024-09-18T01:28:15.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:15.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_39: (2) No such file or directory 2024-09-18T01:28:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_40 2024-09-18T01:28:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_40 deleted 2024-09-18T01:28:15.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_40 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_40 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2024-09-18T01:28:15.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_40: (2) No such file or directory 2024-09-18T01:28:15.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:15.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:15.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_40 deleted '' 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_40 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_40 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_40 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2024-09-18T01:28:15.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:15.351 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_40: (2) No such file or directory 2024-09-18T01:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_41 2024-09-18T01:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_41 deleted 2024-09-18T01:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:28:15.356 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_41 2024-09-18T01:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_41 2024-09-18T01:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2024-09-18T01:28:15.357 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:15.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_41: (2) No such file or directory 2024-09-18T01:28:15.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:15.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:15.392 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:15.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_41 deleted '' 2024-09-18T01:28:15.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:15.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:15.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_41 2024-09-18T01:28:15.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:15.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:15.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:15.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:15.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_41 2024-09-18T01:28:15.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:15.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:15.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_41 2024-09-18T01:28:15.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2024-09-18T01:28:15.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:15.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_41: (2) No such file or directory 2024-09-18T01:28:15.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_42 2024-09-18T01:28:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_42 deleted 2024-09-18T01:28:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:28:15.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_42 2024-09-18T01:28:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:15.535 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:15.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_42 2024-09-18T01:28:15.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2024-09-18T01:28:15.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:15.566 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_42: (2) No such file or directory 2024-09-18T01:28:15.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:15.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:15.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:15.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_42 deleted '' 2024-09-18T01:28:15.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:15.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:15.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_42 2024-09-18T01:28:15.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:15.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:15.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:15.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:15.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_42 2024-09-18T01:28:15.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:15.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:15.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_42 2024-09-18T01:28:15.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2024-09-18T01:28:15.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:15.706 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_42: (2) No such file or directory 2024-09-18T01:28:15.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:15.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:15.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:15.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:15.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:15.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_43 2024-09-18T01:28:15.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_43 deleted 2024-09-18T01:28:15.710 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:15.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:15.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:28:15.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:15.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:15.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:15.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:15.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_43 2024-09-18T01:28:15.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:15.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:15.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_43 2024-09-18T01:28:15.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2024-09-18T01:28:15.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:15.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_43: (2) No such file or directory 2024-09-18T01:28:15.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:15.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:15.749 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_43 deleted '' 2024-09-18T01:28:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:15.851 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_43 2024-09-18T01:28:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:15.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_43 2024-09-18T01:28:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_43 2024-09-18T01:28:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2024-09-18T01:28:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:15.885 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_43: (2) No such file or directory 2024-09-18T01:28:15.889 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_44 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_44 deleted 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:15.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_44 2024-09-18T01:28:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_44 2024-09-18T01:28:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2024-09-18T01:28:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:15.921 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_44: (2) No such file or directory 2024-09-18T01:28:15.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:15.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:15.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:16.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_44 deleted '' 2024-09-18T01:28:16.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_44 2024-09-18T01:28:16.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:16.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_44 2024-09-18T01:28:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_44 2024-09-18T01:28:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2024-09-18T01:28:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_44: (2) No such file or directory 2024-09-18T01:28:16.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_45 2024-09-18T01:28:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_45 deleted 2024-09-18T01:28:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:28:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:16.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_45 2024-09-18T01:28:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_45 2024-09-18T01:28:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2024-09-18T01:28:16.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_45: (2) No such file or directory 2024-09-18T01:28:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:16.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:16.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_45 deleted '' 2024-09-18T01:28:16.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.204 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_45 2024-09-18T01:28:16.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:16.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:16.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:16.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_45 2024-09-18T01:28:16.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_45 2024-09-18T01:28:16.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2024-09-18T01:28:16.205 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_45: (2) No such file or directory 2024-09-18T01:28:16.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:16.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:16.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:16.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:16.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_46 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_46 deleted 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_46 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_46 2024-09-18T01:28:16.242 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2024-09-18T01:28:16.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_46: (2) No such file or directory 2024-09-18T01:28:16.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:16.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:16.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:16.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_46 deleted '' 2024-09-18T01:28:16.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_46 2024-09-18T01:28:16.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:16.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:16.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:16.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_46 2024-09-18T01:28:16.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_46 2024-09-18T01:28:16.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2024-09-18T01:28:16.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_46: (2) No such file or directory 2024-09-18T01:28:16.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:16.419 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_47 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_47 deleted 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_47 2024-09-18T01:28:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_47 2024-09-18T01:28:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2024-09-18T01:28:16.421 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_47: (2) No such file or directory 2024-09-18T01:28:16.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:16.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:16.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:16.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_47 deleted '' 2024-09-18T01:28:16.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_47 2024-09-18T01:28:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_47 2024-09-18T01:28:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_47 2024-09-18T01:28:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2024-09-18T01:28:16.558 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.590 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_47: (2) No such file or directory 2024-09-18T01:28:16.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:16.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_48 2024-09-18T01:28:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_48 deleted 2024-09-18T01:28:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:28:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:16.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_48 2024-09-18T01:28:16.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_48 2024-09-18T01:28:16.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2024-09-18T01:28:16.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_48: (2) No such file or directory 2024-09-18T01:28:16.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:16.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:16.631 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_48 deleted '' 2024-09-18T01:28:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_48 2024-09-18T01:28:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_48 2024-09-18T01:28:16.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_48 2024-09-18T01:28:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2024-09-18T01:28:16.734 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.767 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_48: (2) No such file or directory 2024-09-18T01:28:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:16.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_49 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_49 deleted 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_49 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_49 2024-09-18T01:28:16.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2024-09-18T01:28:16.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_49: (2) No such file or directory 2024-09-18T01:28:16.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:16.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:16.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_49 deleted '' 2024-09-18T01:28:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_49 2024-09-18T01:28:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_49 2024-09-18T01:28:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_49 2024-09-18T01:28:16.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2024-09-18T01:28:16.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_49: (2) No such file or directory 2024-09-18T01:28:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:16.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_50 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_50 deleted 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_50 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_50 2024-09-18T01:28:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2024-09-18T01:28:16.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:16.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_50: (2) No such file or directory 2024-09-18T01:28:16.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:16.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:16.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_50 deleted '' 2024-09-18T01:28:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_50 2024-09-18T01:28:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_50 2024-09-18T01:28:17.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_50 2024-09-18T01:28:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2024-09-18T01:28:17.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:17.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_50: (2) No such file or directory 2024-09-18T01:28:17.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:17.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:17.118 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:17.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:17.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ testlog 'TEST: delete images during bootstrap' 2024-09-18T01:28:17.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ tee -a /tmp/tmp.yeNdCEAq6i/rbd-mirror.test.log 2024-09-18T01:28:17.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ date '+%F %T' 2024-09-18T01:28:17.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo 2024-09-18 01:28:17 TEST: delete images during bootstrap 2024-09-18T01:28:17.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:2024-09-18 01:28:17 TEST: delete images during bootstrap 2024-09-18T01:28:17.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set_pool_mirror_mode cluster1 mirror image 2024-09-18T01:28:17.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:17.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:17.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=image 2024-09-18T01:28:17.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster1 mirror pool enable mirror image 2024-09-18T01:28:17.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd: mirroring is already configured for image mode 2024-09-18T01:28:17.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set_pool_mirror_mode cluster2 mirror image 2024-09-18T01:28:17.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:17.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:17.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=image 2024-09-18T01:28:17.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2024-09-18T01:28:17.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd: mirroring is already configured for image mode 2024-09-18T01:28:17.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ start_mirror cluster1 2024-09-18T01:28:17.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:17.191 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set_cluster_instance cluster1 cluster instance 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local val=cluster1 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster_var_name=cluster 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local instance_var_name=instance 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cluster=cluster1 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ instance=cluster1 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' cluster1 = cluster1 ']' 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ instance=0 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval cluster=cluster1 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ cluster=cluster1 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ eval instance=0 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ instance=0 2024-09-18T01:28:17.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=test 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 1 10 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_1 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_1 512M 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:28:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:28:17.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 512M ']' 2024-09-18T01:28:17.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=512M 2024-09-18T01:28:17.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:28:17.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_1 2024-09-18T01:28:17.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_1 2024-09-18T01:28:17.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:17.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:17.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:17.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:28:17.239 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2024-09-18T01:28:18.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:28:18.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2024-09-18T01:28:18.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_1': 2024-09-18T01:28:18.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:28:18.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:28:18.071 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 185e7372403b 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.185e7372403b 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:28:17 2024 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:28:17 2024 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:28:17 2024 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: a3beaada-11fd-469b-82ba-7a5bda75be42 2024-09-18T01:28:18.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:28:18.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_1 2024-09-18T01:28:18.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:18.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:18.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:18.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:28:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=9 2024-09-18T01:28:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:28:18.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:28:18.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 34590720 bytes 2024-09-18T01:28:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 112064512 bytes 2024-09-18T01:28:18.527 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 155720704 bytes 2024-09-18T01:28:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 211463168 bytes 2024-09-18T01:28:18.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 219238400 bytes 2024-09-18T01:28:18.867 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 258094080 bytes 2024-09-18T01:28:18.994 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 311351296 bytes 2024-09-18T01:28:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 346726400 bytes 2024-09-18T01:28:19.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 425451520 bytes 2024-09-18T01:28:19.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 458911744 bytes 2024-09-18T01:28:19.451 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 459528192 bytes 2024-09-18T01:28:19.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 478522368 bytes 2024-09-18T01:28:19.639 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6481640159024774) > weight (0.25) 2024-09-18T01:28:19.639 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3813638301188693) > weight (0.25) 2024-09-18T01:28:19.639 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-09-18T01:28:19.639 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-09-18T01:28:19.640 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-18T01:28:19.640 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-09-18T01:28:19.640 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:28:19.641 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:28:19.640+0000 7ff6a50f2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 37197) UID: 1000 2024-09-18T01:28:19.655 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 492598272 bytes 2024-09-18T01:28:19.827 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 576099328 bytes 2024-09-18T01:28:19.961 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 625857536 bytes 2024-09-18T01:28:20.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 648824832 bytes 2024-09-18T01:28:20.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 659969024 bytes 2024-09-18T01:28:20.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 685206528 bytes 2024-09-18T01:28:20.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 705070080 bytes 2024-09-18T01:28:20.534 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 763547648 bytes 2024-09-18T01:28:20.630 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 781303808 bytes 2024-09-18T01:28:20.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 800247808 bytes 2024-09-18T01:28:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 890397696 bytes 2024-09-18T01:28:21.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 905328640 bytes 2024-09-18T01:28:21.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 931041280 bytes 2024-09-18T01:28:21.416 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 967362560 bytes 2024-09-18T01:28:21.522 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 998391808 bytes 2024-09-18T01:28:21.641 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1030660096 bytes 2024-09-18T01:28:21.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1086800896 bytes 2024-09-18T01:28:21.903 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1148067840 bytes 2024-09-18T01:28:22.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1180848128 bytes 2024-09-18T01:28:22.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1188840448 bytes 2024-09-18T01:28:22.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1247496192 bytes 2024-09-18T01:28:22.390 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1288271872 bytes 2024-09-18T01:28:22.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1334675456 bytes 2024-09-18T01:28:22.658 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1393393664 bytes 2024-09-18T01:28:22.759 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1419304960 bytes 2024-09-18T01:28:22.846 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1426642944 bytes 2024-09-18T01:28:23.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1536265216 bytes 2024-09-18T01:28:23.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1609711616 bytes 2024-09-18T01:28:23.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1647372288 bytes 2024-09-18T01:28:23.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1647998976 bytes 2024-09-18T01:28:23.548 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1731132416 bytes 2024-09-18T01:28:23.660 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1762593792 bytes 2024-09-18T01:28:23.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1807836160 bytes 2024-09-18T01:28:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1854740480 bytes 2024-09-18T01:28:24.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1871619072 bytes 2024-09-18T01:28:24.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1934794752 bytes 2024-09-18T01:28:24.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1935410176 bytes 2024-09-18T01:28:24.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1974429696 bytes 2024-09-18T01:28:24.502 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2028930048 bytes 2024-09-18T01:28:24.665 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2141038592 bytes 2024-09-18T01:28:24.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2191360000 bytes 2024-09-18T01:28:24.916 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2204640256 bytes 2024-09-18T01:28:25.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2254418944 bytes 2024-09-18T01:28:25.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2339865600 bytes 2024-09-18T01:28:25.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2442698752 bytes 2024-09-18T01:28:25.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2469958656 bytes 2024-09-18T01:28:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2496248832 bytes 2024-09-18T01:28:25.742 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2024-09-18T01:28:25.742 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-18T01:28:25.742 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-18T01:28:25.745 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-18T01:28:25.745 INFO:tasks.rbd_mirror_thrash:waiting for 60 secs before thrashing 2024-09-18T01:28:25.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2544280576 bytes 2024-09-18T01:28:25.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2591176704 bytes 2024-09-18T01:28:26.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2646162432 bytes 2024-09-18T01:28:26.215 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2737168384 bytes 2024-09-18T01:28:26.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2795498496 bytes 2024-09-18T01:28:26.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2806872064 bytes 2024-09-18T01:28:26.573 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2818876416 bytes 2024-09-18T01:28:26.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2838738944 bytes 2024-09-18T01:28:26.804 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2889969664 bytes 2024-09-18T01:28:26.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2912584704 bytes 2024-09-18T01:28:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2948041728 bytes 2024-09-18T01:28:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:28:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:28:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:28:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_1 present 2024-09-18T01:28:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:27.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=present 2024-09-18T01:28:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_1 2024-09-18T01:28:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_1 2024-09-18T01:28:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-18T01:28:27.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:27.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id=11c393f57921 2024-09-18T01:28:27.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:27.128 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:27.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_1 present 11c393f57921 2024-09-18T01:28:27.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=present 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id=11c393f57921 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_1 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_1 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-18T01:28:27.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id=11c393f57921 2024-09-18T01:28:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 11c393f57921 2024-09-18T01:28:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -z 11c393f57921 -o 11c393f57921 = 11c393f57921 2024-09-18T01:28:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=present 2024-09-18T01:28:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test present = present 2024-09-18T01:28:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ disable_mirror cluster2 mirror image_1 2024-09-18T01:28:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:27.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_1 2024-09-18T01:28:28.029 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:28:28.028+0000 7f3640e8e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef4524400 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=185 2024-09-18T01:28:28.029 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:28:28.028+0000 7f3640e8e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef4524400 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=185, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b28075f7-24f5-441c-873b-85d148cb477b, primary_snap_id=b9, last_copied_object_number=128, snap_seqs={185=18446744073709551614}] 2024-09-18T01:28:33.031 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring disabled 2024-09-18T01:28:33.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_1 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_1 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-18T01:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:33.059 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:28:33.058+0000 7f68d0b16640 -1 rbd::mirror::ImageMap: 0x562b53a73320 handle_peer_ack_remove: failed to remove global_image_id=a3beaada-11fd-469b-82ba-7a5bda75be42 2024-09-18T01:28:33.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-09-18T01:28:33.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:28:33.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:28:33.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:28:33.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2024-09-18T01:28:33.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:28:33.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:33.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:33.182 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:28:33.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:28:33.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:28:33.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:28:33.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_1 2024-09-18T01:28:33.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:28:33.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:28:33.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_1 2024-09-18T01:28:33.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-09-18T01:28:33.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:28:33.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-09-18T01:28:33.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:28:33.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:28:33.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:28:33.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:33.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ purge_snapshots cluster2 mirror image_1 2024-09-18T01:28:33.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:33.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:33.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:33.222 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap purge mirror/image_1 2024-09-18T01:28:33.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_1 2024-09-18T01:28:33.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:33.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:33.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:33.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:28:33.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:28:33.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_1 2024-09-18T01:28:33.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:33.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:33.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_1 2024-09-18T01:28:33.268 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2024-09-18T01:28:33.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2024-09-18T01:28:39.572 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9827147308875079) > weight (0.25) 2024-09-18T01:28:39.573 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7335811316418176) > weight (0.25) 2024-09-18T01:28:39.573 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-09-18T01:28:39.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-09-18T01:28:39.573 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-09-18T01:28:39.574 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:28:39.573+0000 7f637dca9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 45116) UID: 1000 2024-09-18T01:28:48.574 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-09-18T01:28:48.574 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:28:48.574 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-09-18T01:28:48.574 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-18T01:28:48.575 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-18T01:28:48.578 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-18T01:28:48.578 INFO:tasks.rbd_mirror_thrash:waiting for 67 secs before thrashing 2024-09-18T01:28:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:28:57.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:28:57.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:28:57.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_2 2024-09-18T01:28:57.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_2 512M 2024-09-18T01:28:57.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:57.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:28:57.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:57.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:28:57.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:28:57.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:28:57.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:28:57.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 512M ']' 2024-09-18T01:28:57.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=512M 2024-09-18T01:28:57.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:28:57.234 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_2 2024-09-18T01:28:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_2 2024-09-18T01:28:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:28:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:28:57.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2024-09-18T01:28:58.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:28:58.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2024-09-18T01:28:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_2': 2024-09-18T01:28:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:28:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:28:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:28:58.080 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 187e1a516525 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.187e1a516525 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:28:57 2024 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:28:57 2024 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:28:57 2024 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 962345d8-c31f-4204-87e2-444578fb455a 2024-09-18T01:28:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:28:58.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_2 2024-09-18T01:28:58.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:28:58.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:28:58.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:28:58.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:28:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=7 2024-09-18T01:28:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:28:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:28:58.226 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 5838848 bytes 2024-09-18T01:28:58.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 45304832 bytes 2024-09-18T01:28:58.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 104304640 bytes 2024-09-18T01:28:58.619 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 168113152 bytes 2024-09-18T01:28:58.761 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 234598400 bytes 2024-09-18T01:28:58.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 286778368 bytes 2024-09-18T01:28:59.007 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 324772864 bytes 2024-09-18T01:28:59.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 376116224 bytes 2024-09-18T01:28:59.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 418961408 bytes 2024-09-18T01:28:59.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 439514112 bytes 2024-09-18T01:28:59.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 468084736 bytes 2024-09-18T01:28:59.588 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 497013760 bytes 2024-09-18T01:28:59.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 582027264 bytes 2024-09-18T01:28:59.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 729600000 bytes 2024-09-18T01:29:00.061 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 753430528 bytes 2024-09-18T01:29:00.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 796870656 bytes 2024-09-18T01:29:00.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 802423808 bytes 2024-09-18T01:29:00.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 833805312 bytes 2024-09-18T01:29:00.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 834432000 bytes 2024-09-18T01:29:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 892626944 bytes 2024-09-18T01:29:00.738 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 934877184 bytes 2024-09-18T01:29:00.890 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 996923392 bytes 2024-09-18T01:29:00.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1010834432 bytes 2024-09-18T01:29:01.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1064314880 bytes 2024-09-18T01:29:01.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1064913920 bytes 2024-09-18T01:29:01.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1065536512 bytes 2024-09-18T01:29:01.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1083724800 bytes 2024-09-18T01:29:01.540 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1162492928 bytes 2024-09-18T01:29:01.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1178417152 bytes 2024-09-18T01:29:01.770 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1219646464 bytes 2024-09-18T01:29:01.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1220253696 bytes 2024-09-18T01:29:01.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1273618432 bytes 2024-09-18T01:29:02.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1329088512 bytes 2024-09-18T01:29:02.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1363195904 bytes 2024-09-18T01:29:02.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1390011392 bytes 2024-09-18T01:29:02.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1444844544 bytes 2024-09-18T01:29:02.604 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1483380736 bytes 2024-09-18T01:29:02.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1541360640 bytes 2024-09-18T01:29:02.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1546937344 bytes 2024-09-18T01:29:03.008 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1644986368 bytes 2024-09-18T01:29:03.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1671567360 bytes 2024-09-18T01:29:03.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1698613248 bytes 2024-09-18T01:29:03.363 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1774004224 bytes 2024-09-18T01:29:03.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1832652800 bytes 2024-09-18T01:29:03.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1870533632 bytes 2024-09-18T01:29:03.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1890399232 bytes 2024-09-18T01:29:03.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1937015808 bytes 2024-09-18T01:29:03.977 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1959831552 bytes 2024-09-18T01:29:04.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1981917184 bytes 2024-09-18T01:29:04.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2008812544 bytes 2024-09-18T01:29:04.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2027918336 bytes 2024-09-18T01:29:04.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2050657280 bytes 2024-09-18T01:29:04.545 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2109107200 bytes 2024-09-18T01:29:04.695 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2169949184 bytes 2024-09-18T01:29:04.777 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2170558464 bytes 2024-09-18T01:29:04.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2194777088 bytes 2024-09-18T01:29:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2260212736 bytes 2024-09-18T01:29:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:29:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:29:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:29:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:29:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_2 present 2024-09-18T01:29:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:29:05.091 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:29:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=present 2024-09-18T01:29:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:29:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:29:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:29:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_2 2024-09-18T01:29:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:29:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:29:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_2 2024-09-18T01:29:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-18T01:29:05.092 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:29:05.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id=11c3a8d9cc30 2024-09-18T01:29:05.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:29:05.338 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:29:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_2 present 11c3a8d9cc30 2024-09-18T01:29:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:29:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:29:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=present 2024-09-18T01:29:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id=11c3a8d9cc30 2024-09-18T01:29:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:29:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:29:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_2 2024-09-18T01:29:05.439 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:29:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:29:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_2 2024-09-18T01:29:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-18T01:29:05.440 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:29:05.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id=11c3a8d9cc30 2024-09-18T01:29:05.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 11c3a8d9cc30 2024-09-18T01:29:05.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -z 11c3a8d9cc30 -o 11c3a8d9cc30 = 11c3a8d9cc30 2024-09-18T01:29:05.483 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=present 2024-09-18T01:29:05.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test present = present 2024-09-18T01:29:05.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:29:05.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ disable_mirror cluster2 mirror image_2 2024-09-18T01:29:05.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:29:05.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:05.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:29:05.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_2 2024-09-18T01:29:06.030 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:29:06.029+0000 7f3640e8e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5ec9c00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=186 2024-09-18T01:29:06.030 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:29:06.029+0000 7f3640e8e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5ec9c00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=186, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b28075f7-24f5-441c-873b-85d148cb477b, primary_snap_id=ba, last_copied_object_number=128, snap_seqs={186=18446744073709551614}] 2024-09-18T01:29:11.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring disabled 2024-09-18T01:29:11.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2024-09-18T01:29:11.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:29:11.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:11.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:29:11.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:29:11.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:29:11.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:29:11.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:29:11.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_2 2024-09-18T01:29:11.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:29:11.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:29:11.044 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_2 2024-09-18T01:29:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-18T01:29:11.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:29:11.084 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-18T01:29:11.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:29:11.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:29:11.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:29:11.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2024-09-18T01:29:11.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_2 2024-09-18T01:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_2 2024-09-18T01:29:11.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-09-18T01:29:11.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:29:11.224 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-09-18T01:29:11.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:29:11.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:29:11.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:29:11.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:29:11.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ purge_snapshots cluster2 mirror image_2 2024-09-18T01:29:11.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:29:11.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:11.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:29:11.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap purge mirror/image_2 2024-09-18T01:29:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_2 2024-09-18T01:29:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:29:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:29:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:29:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:29:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_2 2024-09-18T01:29:11.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:29:11.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:11.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_2 2024-09-18T01:29:11.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-09-18T01:29:11.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-09-18T01:29:25.746 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.41496034110596103) > weight (0.25) 2024-09-18T01:29:25.747 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.648551512154112) > weight (0.25) 2024-09-18T01:29:25.747 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.6009991532139856) > weight (0.25) 2024-09-18T01:29:25.747 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6124518809278351) > weight (0.25) 2024-09-18T01:29:25.747 INFO:tasks.rbd_mirror_thrash:waiting for 85 secs before thrashing 2024-09-18T01:29:35.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:29:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:29:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:29:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_3 2024-09-18T01:29:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_3 512M 2024-09-18T01:29:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:29:35.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:29:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:29:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:29:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:29:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:29:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 512M ']' 2024-09-18T01:29:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=512M 2024-09-18T01:29:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:29:35.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_3 2024-09-18T01:29:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_3 2024-09-18T01:29:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:29:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:35.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:29:35.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:29:35.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2024-09-18T01:29:36.077 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:29:36.085 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2024-09-18T01:29:36.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_3': 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 188fe40f74e0 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.188fe40f74e0 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:29:35 2024 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:29:35 2024 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:29:35 2024 2024-09-18T01:29:36.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:29:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:29:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 3946b1b2-74cb-4e53-a74d-a4c79d38e748 2024-09-18T01:29:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:29:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_3 2024-09-18T01:29:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:29:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:29:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:29:36.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:29:36.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:29:36.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:29:36.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 34965504 bytes 2024-09-18T01:29:36.405 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 83054592 bytes 2024-09-18T01:29:36.594 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 192638976 bytes 2024-09-18T01:29:36.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 236551168 bytes 2024-09-18T01:29:36.815 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 256223232 bytes 2024-09-18T01:29:36.919 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 282078208 bytes 2024-09-18T01:29:37.062 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 349523968 bytes 2024-09-18T01:29:37.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 399000576 bytes 2024-09-18T01:29:37.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 462733312 bytes 2024-09-18T01:29:37.418 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 475218944 bytes 2024-09-18T01:29:37.555 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 538545152 bytes 2024-09-18T01:29:37.671 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 573104128 bytes 2024-09-18T01:29:37.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 599012352 bytes 2024-09-18T01:29:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 655134720 bytes 2024-09-18T01:29:38.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 708179968 bytes 2024-09-18T01:29:38.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 743654400 bytes 2024-09-18T01:29:38.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 846530560 bytes 2024-09-18T01:29:38.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 924690432 bytes 2024-09-18T01:29:38.629 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 998050816 bytes 2024-09-18T01:29:38.731 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 1011325952 bytes 2024-09-18T01:29:38.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 1025420288 bytes 2024-09-18T01:29:38.944 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1050184704 bytes 2024-09-18T01:29:39.066 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1117743104 bytes 2024-09-18T01:29:39.223 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1161368576 bytes 2024-09-18T01:29:39.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1228347392 bytes 2024-09-18T01:29:39.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1257078784 bytes 2024-09-18T01:29:39.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1328192512 bytes 2024-09-18T01:29:39.771 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1382144000 bytes 2024-09-18T01:29:39.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1434365952 bytes 2024-09-18T01:29:40.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1489657856 bytes 2024-09-18T01:29:40.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1605602304 bytes 2024-09-18T01:29:40.372 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1663204352 bytes 2024-09-18T01:29:40.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1709069312 bytes 2024-09-18T01:29:40.621 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1748278272 bytes 2024-09-18T01:29:40.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1770161152 bytes 2024-09-18T01:29:40.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1783759872 bytes 2024-09-18T01:29:40.976 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1860808704 bytes 2024-09-18T01:29:41.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1872781312 bytes 2024-09-18T01:29:41.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1904569344 bytes 2024-09-18T01:29:41.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1947233280 bytes 2024-09-18T01:29:41.403 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1959087104 bytes 2024-09-18T01:29:41.560 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 2033495040 bytes 2024-09-18T01:29:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 2059781120 bytes 2024-09-18T01:29:41.819 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 2147618816 bytes 2024-09-18T01:29:41.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 2202389504 bytes 2024-09-18T01:29:42.068 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 2218617856 bytes 2024-09-18T01:29:42.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 2219896832 bytes 2024-09-18T01:29:42.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2255492096 bytes 2024-09-18T01:29:42.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2296212480 bytes 2024-09-18T01:29:42.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2296842240 bytes 2024-09-18T01:29:42.627 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2368220160 bytes 2024-09-18T01:29:42.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2373085184 bytes 2024-09-18T01:29:42.858 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2436737024 bytes 2024-09-18T01:29:42.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2490832896 bytes 2024-09-18T01:29:43.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2495959040 bytes 2024-09-18T01:29:43.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2518810624 bytes 2024-09-18T01:29:43.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2519460864 bytes 2024-09-18T01:29:43.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2520107008 bytes 2024-09-18T01:29:43.490 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2589106176 bytes 2024-09-18T01:29:43.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2656369664 bytes 2024-09-18T01:29:43.803 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2733503488 bytes 2024-09-18T01:29:43.937 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2789441536 bytes 2024-09-18T01:29:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2851127296 bytes 2024-09-18T01:29:44.218 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2893498368 bytes 2024-09-18T01:29:44.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2955452416 bytes 2024-09-18T01:29:44.481 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2986099712 bytes 2024-09-18T01:29:44.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3038839808 bytes 2024-09-18T01:29:44.753 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3090737152 bytes 2024-09-18T01:29:44.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3123178496 bytes 2024-09-18T01:29:44.986 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3149956096 bytes 2024-09-18T01:29:45.082 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3150570496 bytes 2024-09-18T01:29:45.221 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3214034944 bytes 2024-09-18T01:29:45.342 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3247284224 bytes 2024-09-18T01:29:45.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3281603584 bytes 2024-09-18T01:29:45.596 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3331360768 bytes 2024-09-18T01:29:45.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3344753664 bytes 2024-09-18T01:29:45.839 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3387563008 bytes 2024-09-18T01:29:45.987 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3442975744 bytes 2024-09-18T01:29:46.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3504722944 bytes 2024-09-18T01:29:46.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3524992000 bytes 2024-09-18T01:29:46.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3543912448 bytes 2024-09-18T01:29:46.509 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3624968192 bytes 2024-09-18T01:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3636989952 bytes 2024-09-18T01:29:46.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3733669888 bytes 2024-09-18T01:29:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3736717312 bytes 2024-09-18T01:29:46.983 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3744392192 bytes 2024-09-18T01:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3777298432 bytes 2024-09-18T01:29:47.228 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3815232512 bytes 2024-09-18T01:29:47.346 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3844547584 bytes 2024-09-18T01:29:47.458 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3870283776 bytes 2024-09-18T01:29:47.601 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3929065472 bytes 2024-09-18T01:29:47.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3951661056 bytes 2024-09-18T01:29:47.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3989011456 bytes 2024-09-18T01:29:47.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4032741376 bytes 2024-09-18T01:29:48.098 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4071842816 bytes 2024-09-18T01:29:48.201 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4088020992 bytes 2024-09-18T01:29:48.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4136539136 bytes 2024-09-18T01:29:48.458 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4198579200 bytes 2024-09-18T01:29:48.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4266267648 bytes 2024-09-18T01:29:48.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4311857152 bytes 2024-09-18T01:29:48.905 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4378307584 bytes 2024-09-18T01:29:49.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4416419840 bytes 2024-09-18T01:29:49.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4467427328 bytes 2024-09-18T01:29:49.276 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4473337856 bytes 2024-09-18T01:29:49.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4526888960 bytes 2024-09-18T01:29:49.562 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4586181632 bytes 2024-09-18T01:29:49.697 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4638737408 bytes 2024-09-18T01:29:49.822 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4677226496 bytes 2024-09-18T01:29:49.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4714590208 bytes 2024-09-18T01:29:50.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 4771276800 bytes 2024-09-18T01:29:50.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:29:50.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:29:50.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:29:50.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:29:50.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_3 present 2024-09-18T01:29:50.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:29:50.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:50.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:29:50.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=present 2024-09-18T01:29:50.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:29:50.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:29:50.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:29:50.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_3 2024-09-18T01:29:50.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:29:50.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:29:50.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_3 2024-09-18T01:29:50.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-18T01:29:50.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:29:50.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id=11c3bcdeff7f 2024-09-18T01:29:50.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:29:50.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:29:50.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_3 present 11c3bcdeff7f 2024-09-18T01:29:50.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:29:50.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:50.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:29:50.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=present 2024-09-18T01:29:50.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id=11c3bcdeff7f 2024-09-18T01:29:50.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:29:50.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:29:50.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_3 2024-09-18T01:29:50.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:29:50.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:29:50.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_3 2024-09-18T01:29:50.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-18T01:29:50.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:29:50.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id=11c3bcdeff7f 2024-09-18T01:29:50.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 11c3bcdeff7f 2024-09-18T01:29:50.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -z 11c3bcdeff7f -o 11c3bcdeff7f = 11c3bcdeff7f 2024-09-18T01:29:50.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=present 2024-09-18T01:29:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test present = present 2024-09-18T01:29:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:29:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ disable_mirror cluster2 mirror image_3 2024-09-18T01:29:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:29:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:29:50.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_3 2024-09-18T01:29:51.093 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:29:51.091+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef52ae000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=187 2024-09-18T01:29:51.093 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:29:51.091+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef52ae000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=187, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b28075f7-24f5-441c-873b-85d148cb477b, primary_snap_id=bb, last_copied_object_number=128, snap_seqs={187=18446744073709551614}] 2024-09-18T01:29:55.579 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-18T01:29:55.580 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-18T01:29:55.580 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-18T01:29:55.581 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:29:55.580+0000 7facc8318640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 64046) UID: 1000 2024-09-18T01:29:56.094 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring disabled 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_3 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:29:56.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_3 2024-09-18T01:29:56.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-18T01:29:56.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:29:56.144 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-09-18T01:29:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:29:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:29:56.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:29:56.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2024-09-18T01:29:56.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:29:56.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:56.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:29:56.250 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_3 2024-09-18T01:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_3 2024-09-18T01:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-09-18T01:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:29:56.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-09-18T01:29:56.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:29:56.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:29:56.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:29:56.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:29:56.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ purge_snapshots cluster2 mirror image_3 2024-09-18T01:29:56.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:29:56.290 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:56.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:29:56.291 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap purge mirror/image_3 2024-09-18T01:29:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_3 2024-09-18T01:29:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:29:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:29:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:29:56.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:29:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_3 2024-09-18T01:29:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:29:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:29:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_3 2024-09-18T01:29:56.337 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-09-18T01:29:56.380 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-09-18T01:30:02.581 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-18T01:30:02.581 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:30:02.581 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-18T01:30:02.581 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-18T01:30:02.582 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-18T01:30:02.585 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-18T01:30:02.585 INFO:tasks.rbd_mirror_thrash:waiting for 98 secs before thrashing 2024-09-18T01:30:18.461 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:30:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:30:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:30:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_4 2024-09-18T01:30:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_4 512M 2024-09-18T01:30:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:30:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:30:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:30:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:30:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:30:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:30:18.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:30:18.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 512M ']' 2024-09-18T01:30:18.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=512M 2024-09-18T01:30:18.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:30:18.466 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_4 2024-09-18T01:30:18.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_4 2024-09-18T01:30:18.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:30:18.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:30:18.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:30:18.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:30:18.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2024-09-18T01:30:19.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:30:19.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2024-09-18T01:30:19.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_4': 2024-09-18T01:30:19.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:30:19.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 18a5b5335648 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.18a5b5335648 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:30:18 2024 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:30:18 2024 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:30:18 2024 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:30:19.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: e789f3ac-e37f-434f-b266-5056371d2ce0 2024-09-18T01:30:19.104 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:30:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_4 2024-09-18T01:30:19.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:30:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:30:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:30:19.106 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:30:19.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=14 2024-09-18T01:30:19.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:30:19.107 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:30:19.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 53554176 bytes 2024-09-18T01:30:19.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 88348672 bytes 2024-09-18T01:30:19.500 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 104996864 bytes 2024-09-18T01:30:19.628 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 155115520 bytes 2024-09-18T01:30:19.736 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 187567104 bytes 2024-09-18T01:30:19.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 202067968 bytes 2024-09-18T01:30:19.942 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 234577920 bytes 2024-09-18T01:30:20.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 280532992 bytes 2024-09-18T01:30:20.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 346343424 bytes 2024-09-18T01:30:20.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 393881600 bytes 2024-09-18T01:30:20.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 428506112 bytes 2024-09-18T01:30:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 433886208 bytes 2024-09-18T01:30:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 456958976 bytes 2024-09-18T01:30:20.750 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 487237632 bytes 2024-09-18T01:30:20.861 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 545049600 bytes 2024-09-18T01:30:21.015 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 583301120 bytes 2024-09-18T01:30:21.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 619652096 bytes 2024-09-18T01:30:21.243 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 634281984 bytes 2024-09-18T01:30:21.343 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 652711936 bytes 2024-09-18T01:30:21.458 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 688541696 bytes 2024-09-18T01:30:21.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 709199872 bytes 2024-09-18T01:30:21.682 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 741039104 bytes 2024-09-18T01:30:21.828 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 811394048 bytes 2024-09-18T01:30:21.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 862759936 bytes 2024-09-18T01:30:22.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 886828032 bytes 2024-09-18T01:30:22.209 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 947684352 bytes 2024-09-18T01:30:22.316 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 978577408 bytes 2024-09-18T01:30:22.429 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1004209152 bytes 2024-09-18T01:30:22.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1072278528 bytes 2024-09-18T01:30:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1093220352 bytes 2024-09-18T01:30:22.764 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1103232000 bytes 2024-09-18T01:30:22.910 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1162519552 bytes 2024-09-18T01:30:23.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1164452864 bytes 2024-09-18T01:30:23.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1211173888 bytes 2024-09-18T01:30:23.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1246872576 bytes 2024-09-18T01:30:23.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1303657472 bytes 2024-09-18T01:30:23.492 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1328755712 bytes 2024-09-18T01:30:23.600 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1350345728 bytes 2024-09-18T01:30:23.739 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1410506752 bytes 2024-09-18T01:30:23.859 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1447815168 bytes 2024-09-18T01:30:23.964 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1471830016 bytes 2024-09-18T01:30:24.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1562473472 bytes 2024-09-18T01:30:24.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1607376896 bytes 2024-09-18T01:30:24.396 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1664651264 bytes 2024-09-18T01:30:24.507 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1722158080 bytes 2024-09-18T01:30:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1763769344 bytes 2024-09-18T01:30:24.744 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1767750656 bytes 2024-09-18T01:30:24.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1815472128 bytes 2024-09-18T01:30:24.992 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1879957504 bytes 2024-09-18T01:30:25.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1880564736 bytes 2024-09-18T01:30:25.233 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 1933075456 bytes 2024-09-18T01:30:25.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 1997988864 bytes 2024-09-18T01:30:25.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2067837952 bytes 2024-09-18T01:30:25.610 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2068430848 bytes 2024-09-18T01:30:25.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2188367872 bytes 2024-09-18T01:30:25.950 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2260302848 bytes 2024-09-18T01:30:26.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2330566656 bytes 2024-09-18T01:30:26.207 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2340560896 bytes 2024-09-18T01:30:26.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2405602304 bytes 2024-09-18T01:30:26.478 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2456475648 bytes 2024-09-18T01:30:26.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2473612288 bytes 2024-09-18T01:30:26.702 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2532798464 bytes 2024-09-18T01:30:26.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2546716672 bytes 2024-09-18T01:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2584593408 bytes 2024-09-18T01:30:27.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2655933440 bytes 2024-09-18T01:30:27.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2693219328 bytes 2024-09-18T01:30:27.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2770108416 bytes 2024-09-18T01:30:27.550 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2859077632 bytes 2024-09-18T01:30:27.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2944313344 bytes 2024-09-18T01:30:27.887 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2985560064 bytes 2024-09-18T01:30:28.042 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3068261376 bytes 2024-09-18T01:30:28.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3109436416 bytes 2024-09-18T01:30:28.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3172636672 bytes 2024-09-18T01:30:28.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3229569024 bytes 2024-09-18T01:30:28.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3285111808 bytes 2024-09-18T01:30:28.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3335887872 bytes 2024-09-18T01:30:28.872 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3363744768 bytes 2024-09-18T01:30:29.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3429569536 bytes 2024-09-18T01:30:29.145 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3486584832 bytes 2024-09-18T01:30:29.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3525558272 bytes 2024-09-18T01:30:29.389 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3555098624 bytes 2024-09-18T01:30:29.480 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3557735424 bytes 2024-09-18T01:30:29.591 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3586434048 bytes 2024-09-18T01:30:29.717 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3651146752 bytes 2024-09-18T01:30:29.848 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3677176832 bytes 2024-09-18T01:30:30.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3771951104 bytes 2024-09-18T01:30:30.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3796337664 bytes 2024-09-18T01:30:30.259 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3828469760 bytes 2024-09-18T01:30:30.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3890041856 bytes 2024-09-18T01:30:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3919534080 bytes 2024-09-18T01:30:30.635 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3962499072 bytes 2024-09-18T01:30:30.774 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 4018464768 bytes 2024-09-18T01:30:30.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 4060582912 bytes 2024-09-18T01:30:31.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 4103861248 bytes 2024-09-18T01:30:31.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 4130799616 bytes 2024-09-18T01:30:31.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 4222604288 bytes 2024-09-18T01:30:31.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4238222336 bytes 2024-09-18T01:30:31.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4281909248 bytes 2024-09-18T01:30:31.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4350708736 bytes 2024-09-18T01:30:31.821 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4417724416 bytes 2024-09-18T01:30:31.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4442788864 bytes 2024-09-18T01:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4461203456 bytes 2024-09-18T01:30:32.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4475917312 bytes 2024-09-18T01:30:32.235 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4495952896 bytes 2024-09-18T01:30:32.347 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4521748480 bytes 2024-09-18T01:30:32.457 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10600 IOs, 4546258944 bytes 2024-09-18T01:30:32.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10700 IOs, 4590417920 bytes 2024-09-18T01:30:32.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10800 IOs, 4602035200 bytes 2024-09-18T01:30:32.797 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10900 IOs, 4641695744 bytes 2024-09-18T01:30:32.949 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11000 IOs, 4702715904 bytes 2024-09-18T01:30:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:11100 IOs, 4771357696 bytes 2024-09-18T01:30:33.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:30:33.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:30:33.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:30:33.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:30:33.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_4 present 2024-09-18T01:30:33.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:30:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:30:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:30:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=present 2024-09-18T01:30:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:30:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:30:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:30:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_4 2024-09-18T01:30:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:30:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:30:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_4 2024-09-18T01:30:33.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-18T01:30:33.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:30:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id=11c3424f7c4e 2024-09-18T01:30:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:30:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:30:33.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_4 present 11c3424f7c4e 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=present 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id=11c3424f7c4e 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_4 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_4 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-18T01:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:30:33.302 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id=11c3424f7c4e 2024-09-18T01:30:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 11c3424f7c4e 2024-09-18T01:30:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -z 11c3424f7c4e -o 11c3424f7c4e = 11c3424f7c4e 2024-09-18T01:30:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=present 2024-09-18T01:30:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test present = present 2024-09-18T01:30:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:30:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ disable_mirror cluster2 mirror image_4 2024-09-18T01:30:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:30:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:30:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:30:33.303 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_4 2024-09-18T01:30:34.078 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:30:34.076+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef55fb000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=188 2024-09-18T01:30:34.078 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:30:34.076+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef55fb000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=188, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b28075f7-24f5-441c-873b-85d148cb477b, primary_snap_id=bc, last_copied_object_number=128, snap_seqs={188=18446744073709551614}] 2024-09-18T01:30:39.081 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring disabled 2024-09-18T01:30:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2024-09-18T01:30:39.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_4 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_4 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-18T01:30:39.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:30:39.132 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-09-18T01:30:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:30:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:30:39.134 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:30:39.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2024-09-18T01:30:39.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:30:39.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:30:39.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:30:39.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:30:39.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:30:39.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:30:39.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:30:39.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_4 2024-09-18T01:30:39.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:30:39.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:30:39.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_4 2024-09-18T01:30:39.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-09-18T01:30:39.237 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:30:39.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-09-18T01:30:39.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:30:39.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:30:39.278 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:30:39.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:30:39.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ purge_snapshots cluster2 mirror image_4 2024-09-18T01:30:39.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:30:39.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:30:39.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:30:39.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap purge mirror/image_4 2024-09-18T01:30:39.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_4 2024-09-18T01:30:39.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:30:39.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:30:39.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:30:39.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:30:39.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:30:39.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_4 2024-09-18T01:30:39.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:30:39.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:30:39.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_4 2024-09-18T01:30:39.326 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-09-18T01:30:39.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-09-18T01:30:50.748 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-09-18T01:30:50.748 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-09-18T01:30:50.749 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-09-18T01:30:50.750 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:30:50.749+0000 7f774ffec640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 52187) UID: 1000 2024-09-18T01:30:57.750 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-09-18T01:30:57.750 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:30:57.750 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-09-18T01:30:57.750 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-09-18T01:30:57.751 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-09-18T01:30:57.754 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-09-18T01:30:57.754 INFO:tasks.rbd_mirror_thrash:waiting for 77 secs before thrashing 2024-09-18T01:31:03.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:31:03.322 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:31:03.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:31:03.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_5 2024-09-18T01:31:03.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_5 512M 2024-09-18T01:31:03.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:03.323 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:31:03.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:03.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:31:03.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:31:03.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:31:03.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:31:03.324 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 512M ']' 2024-09-18T01:31:03.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=512M 2024-09-18T01:31:03.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:31:03.325 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_5 2024-09-18T01:31:03.369 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_5 2024-09-18T01:31:03.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:03.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:03.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:31:03.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:31:03.370 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2024-09-18T01:31:04.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:31:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2024-09-18T01:31:04.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_5': 2024-09-18T01:31:04.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:31:04.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:31:04.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:31:04.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 18bf2fdd28d9 2024-09-18T01:31:04.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.18bf2fdd28d9 2024-09-18T01:31:04.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:31:04.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:31:04.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:31:04.171 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:31:04.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:31:03 2024 2024-09-18T01:31:04.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:31:03 2024 2024-09-18T01:31:04.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:31:03 2024 2024-09-18T01:31:04.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:31:04.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:31:04.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 4654d7fc-6238-49b4-8f3b-6b34fc394e5d 2024-09-18T01:31:04.172 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:31:04.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_5 2024-09-18T01:31:04.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:04.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:04.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:31:04.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:31:04.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=6 2024-09-18T01:31:04.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:31:04.176 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:31:04.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 22318080 bytes 2024-09-18T01:31:04.434 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 46647296 bytes 2024-09-18T01:31:04.563 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 100791296 bytes 2024-09-18T01:31:04.673 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 130728960 bytes 2024-09-18T01:31:04.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 171155456 bytes 2024-09-18T01:31:04.886 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 179041280 bytes 2024-09-18T01:31:04.984 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 199056384 bytes 2024-09-18T01:31:05.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 214573056 bytes 2024-09-18T01:31:05.199 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 259286016 bytes 2024-09-18T01:31:05.294 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 272849920 bytes 2024-09-18T01:31:05.417 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 321870848 bytes 2024-09-18T01:31:05.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 397998080 bytes 2024-09-18T01:31:05.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 422722560 bytes 2024-09-18T01:31:05.789 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 461432832 bytes 2024-09-18T01:31:05.906 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 505457664 bytes 2024-09-18T01:31:06.053 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 578911232 bytes 2024-09-18T01:31:06.194 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 635750400 bytes 2024-09-18T01:31:06.333 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 695878656 bytes 2024-09-18T01:31:06.425 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 703918080 bytes 2024-09-18T01:31:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 752318464 bytes 2024-09-18T01:31:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 869601280 bytes 2024-09-18T01:31:07.054 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 892177408 bytes 2024-09-18T01:31:07.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 899325952 bytes 2024-09-18T01:31:07.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 916337664 bytes 2024-09-18T01:31:07.364 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 958025728 bytes 2024-09-18T01:31:07.495 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1012018176 bytes 2024-09-18T01:31:07.656 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1081349120 bytes 2024-09-18T01:31:07.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1135645696 bytes 2024-09-18T01:31:07.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1155579904 bytes 2024-09-18T01:31:08.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1197091840 bytes 2024-09-18T01:31:08.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1220021248 bytes 2024-09-18T01:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1268940800 bytes 2024-09-18T01:31:08.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1269551104 bytes 2024-09-18T01:31:08.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1320757248 bytes 2024-09-18T01:31:08.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1406483456 bytes 2024-09-18T01:31:08.733 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1435467776 bytes 2024-09-18T01:31:08.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1479808000 bytes 2024-09-18T01:31:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1480449024 bytes 2024-09-18T01:31:09.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1502780416 bytes 2024-09-18T01:31:09.166 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1539358720 bytes 2024-09-18T01:31:09.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1593686016 bytes 2024-09-18T01:31:09.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1650614272 bytes 2024-09-18T01:31:09.617 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1754351616 bytes 2024-09-18T01:31:09.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1803331584 bytes 2024-09-18T01:31:09.840 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1809691648 bytes 2024-09-18T01:31:09.972 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1863495680 bytes 2024-09-18T01:31:10.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1872217088 bytes 2024-09-18T01:31:10.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:31:10.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:31:10.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:31:10.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:31:10.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_5 present 2024-09-18T01:31:10.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:31:10.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:10.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:31:10.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=present 2024-09-18T01:31:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:31:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:31:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:31:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_5 2024-09-18T01:31:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:31:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:31:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_5 2024-09-18T01:31:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-18T01:31:10.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:31:10.229 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id=11c366c3229f 2024-09-18T01:31:10.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:31:10.230 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:31:10.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_5 present 11c366c3229f 2024-09-18T01:31:10.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:31:10.331 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=present 2024-09-18T01:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id=11c366c3229f 2024-09-18T01:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_5 2024-09-18T01:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_5 2024-09-18T01:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-18T01:31:10.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:31:10.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id=11c366c3229f 2024-09-18T01:31:10.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 11c366c3229f 2024-09-18T01:31:10.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -z 11c366c3229f -o 11c366c3229f = 11c366c3229f 2024-09-18T01:31:10.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=present 2024-09-18T01:31:10.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test present = present 2024-09-18T01:31:10.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:31:10.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ disable_mirror cluster2 mirror image_5 2024-09-18T01:31:10.376 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:10.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:10.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:31:10.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_5 2024-09-18T01:31:10.636 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:31:10.635+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5e81400 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=189 2024-09-18T01:31:10.637 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:31:10.635+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5e81400 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=189, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b28075f7-24f5-441c-873b-85d148cb477b, primary_snap_id=bd, last_copied_object_number=128, snap_seqs={189=18446744073709551614}] 2024-09-18T01:31:15.640 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring disabled 2024-09-18T01:31:15.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2024-09-18T01:31:15.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:31:15.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:15.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:31:15.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:31:15.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:31:15.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:31:15.649 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:31:15.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_5 2024-09-18T01:31:15.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:31:15.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:31:15.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_5 2024-09-18T01:31:15.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-18T01:31:15.650 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:31:15.672 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:31:15.671+0000 7f68d1317640 -1 rbd::mirror::ImageMap: 0x562b53a73320 handle_peer_ack_remove: failed to remove global_image_id=4654d7fc-6238-49b4-8f3b-6b34fc394e5d 2024-09-18T01:31:15.688 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-18T01:31:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:31:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:31:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:31:15.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2024-09-18T01:31:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:31:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:31:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:31:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:31:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:31:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:31:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_5 2024-09-18T01:31:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:31:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:31:15.794 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_5 2024-09-18T01:31:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-09-18T01:31:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:31:15.829 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-09-18T01:31:15.833 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:31:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:31:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:31:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:31:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ purge_snapshots cluster2 mirror image_5 2024-09-18T01:31:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:31:15.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap purge mirror/image_5 2024-09-18T01:31:15.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_5 2024-09-18T01:31:15.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:15.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:15.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:31:15.878 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:31:15.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:31:15.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_5 2024-09-18T01:31:15.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:15.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:15.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_5 2024-09-18T01:31:15.880 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-09-18T01:31:15.924 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-09-18T01:31:40.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:31:40.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:31:40.449 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_6 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_6 512M 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 512M ']' 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=512M 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:31:40.450 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_6 2024-09-18T01:31:40.496 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_6 2024-09-18T01:31:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:31:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:31:40.497 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2024-09-18T01:31:40.586 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-18T01:31:40.586 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-18T01:31:40.586 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-09-18T01:31:40.587 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:31:40.587+0000 7f8b7a02b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 69409) UID: 1000 2024-09-18T01:31:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:31:41.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2024-09-18T01:31:41.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_6': 2024-09-18T01:31:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:31:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:31:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:31:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 18d0c2a6548e 2024-09-18T01:31:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.18d0c2a6548e 2024-09-18T01:31:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:31:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:31:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:31:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:31:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:31:40 2024 2024-09-18T01:31:41.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:31:40 2024 2024-09-18T01:31:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:31:40 2024 2024-09-18T01:31:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:31:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:31:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 39b57cf7-6f1a-478c-91e7-b5e7debbc1ba 2024-09-18T01:31:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:31:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_6 2024-09-18T01:31:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:31:41.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:31:41.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=6 2024-09-18T01:31:41.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:31:41.105 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:31:41.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 27941888 bytes 2024-09-18T01:31:41.345 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 35620864 bytes 2024-09-18T01:31:41.459 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 67888128 bytes 2024-09-18T01:31:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 118535168 bytes 2024-09-18T01:31:41.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 157731840 bytes 2024-09-18T01:31:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 184041472 bytes 2024-09-18T01:31:42.096 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 201399296 bytes 2024-09-18T01:31:42.185 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 222235648 bytes 2024-09-18T01:31:42.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 244595712 bytes 2024-09-18T01:31:42.436 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 295792640 bytes 2024-09-18T01:31:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 360915968 bytes 2024-09-18T01:31:42.735 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 437637120 bytes 2024-09-18T01:31:42.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 469013504 bytes 2024-09-18T01:31:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 469621760 bytes 2024-09-18T01:31:43.025 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 488973312 bytes 2024-09-18T01:31:43.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 502687744 bytes 2024-09-18T01:31:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 512230400 bytes 2024-09-18T01:31:43.525 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 543426560 bytes 2024-09-18T01:31:43.714 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 658669568 bytes 2024-09-18T01:31:43.865 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 665484288 bytes 2024-09-18T01:31:43.980 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 716425216 bytes 2024-09-18T01:31:44.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 717030400 bytes 2024-09-18T01:31:44.196 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 755996672 bytes 2024-09-18T01:31:44.375 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 863779840 bytes 2024-09-18T01:31:44.587 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-18T01:31:44.587 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:31:44.588 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-18T01:31:44.588 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-18T01:31:44.588 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-18T01:31:44.591 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-18T01:31:44.591 INFO:tasks.rbd_mirror_thrash:waiting for 88 secs before thrashing 2024-09-18T01:31:44.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 876658688 bytes 2024-09-18T01:31:44.799 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 900981760 bytes 2024-09-18T01:31:44.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 955556864 bytes 2024-09-18T01:31:45.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 967620608 bytes 2024-09-18T01:31:45.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1017690112 bytes 2024-09-18T01:31:45.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1072343040 bytes 2024-09-18T01:31:45.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1079866368 bytes 2024-09-18T01:31:45.526 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1141065728 bytes 2024-09-18T01:31:45.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1192412160 bytes 2024-09-18T01:31:45.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1228413952 bytes 2024-09-18T01:31:45.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1270519808 bytes 2024-09-18T01:31:46.019 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1288529920 bytes 2024-09-18T01:31:46.174 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1368302592 bytes 2024-09-18T01:31:46.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1446268928 bytes 2024-09-18T01:31:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1462222848 bytes 2024-09-18T01:31:46.513 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1462832128 bytes 2024-09-18T01:31:46.685 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1556701184 bytes 2024-09-18T01:31:46.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1565066240 bytes 2024-09-18T01:31:46.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1579067392 bytes 2024-09-18T01:31:47.029 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1664428032 bytes 2024-09-18T01:31:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:31:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:31:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:31:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:31:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_6 present 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=present 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_6 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_6 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-18T01:31:47.112 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:31:47.155 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id=11c327a8dd54 2024-09-18T01:31:47.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:31:47.192 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:31:47.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_6 present 11c327a8dd54 2024-09-18T01:31:47.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:31:47.256 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:31:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=present 2024-09-18T01:31:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id=11c327a8dd54 2024-09-18T01:31:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:31:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:31:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_6 2024-09-18T01:31:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:31:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:31:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_6 2024-09-18T01:31:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-18T01:31:47.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:31:47.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id=11c327a8dd54 2024-09-18T01:31:47.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 11c327a8dd54 2024-09-18T01:31:47.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -z 11c327a8dd54 -o 11c327a8dd54 = 11c327a8dd54 2024-09-18T01:31:47.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=present 2024-09-18T01:31:47.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test present = present 2024-09-18T01:31:47.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:31:47.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ disable_mirror cluster2 mirror image_6 2024-09-18T01:31:47.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:47.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:47.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:31:47.301 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_6 2024-09-18T01:31:48.065 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:31:48.064+0000 7f80d5d16640 -1 reset not still connected to 0x5641831faf70 2024-09-18T01:31:48.065 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:31:48.064+0000 7f80d5d16640 -1 reset not still connected to 0x564195d73520 2024-09-18T01:31:48.065 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:31:48.064+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5369c00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=190 2024-09-18T01:31:48.066 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:31:48.064+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5369c00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=190, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b28075f7-24f5-441c-873b-85d148cb477b, primary_snap_id=be, last_copied_object_number=128, snap_seqs={190=18446744073709551614}] 2024-09-18T01:31:48.066 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:31:48.064+0000 7f80d5d16640 -1 reset not still connected to 0x56419e3251e0 2024-09-18T01:31:48.066 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:31:48.065+0000 7f80d5d16640 -1 reset not still connected to 0x5641a3f24750 2024-09-18T01:31:50.481 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:31:50.480+0000 7ffa51edc640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ccedad3ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:31:50.481 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:31:50.480+0000 7ffa586e9640 -1 rbd::mirror::ImageReplayer: 0x55ccee22e280 [3/39b57cf7-6f1a-478c-91e7-b5e7debbc1ba] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:31:50.753 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:31:50.752+0000 7ffa516db640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55cced9bfa00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:31:50.753 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:31:50.752+0000 7ffa57ee8640 -1 rbd::mirror::ImageReplayer: 0x55ccee22e280 [3/39b57cf7-6f1a-478c-91e7-b5e7debbc1ba] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:31:53.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring disabled 2024-09-18T01:31:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2024-09-18T01:31:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:31:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:31:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:31:53.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:31:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:31:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:31:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_6 2024-09-18T01:31:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:31:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:31:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_6 2024-09-18T01:31:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-18T01:31:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:31:53.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-09-18T01:31:53.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:31:53.114 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:31:53.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:31:53.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2024-09-18T01:31:53.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:31:53.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:53.216 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:31:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:31:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:31:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:31:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:31:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_6 2024-09-18T01:31:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:31:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:31:53.217 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_6 2024-09-18T01:31:53.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-09-18T01:31:53.220 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:31:53.253 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-09-18T01:31:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:31:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:31:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:31:53.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:31:53.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ purge_snapshots cluster2 mirror image_6 2024-09-18T01:31:53.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:53.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:53.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:31:53.258 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap purge mirror/image_6 2024-09-18T01:31:53.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_6 2024-09-18T01:31:53.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:53.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:53.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:31:53.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:31:53.305 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:31:53.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_6 2024-09-18T01:31:53.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:31:53.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:31:53.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_6 2024-09-18T01:31:53.306 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-09-18T01:31:53.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-09-18T01:32:14.755 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8893141038313193) > weight (0.25) 2024-09-18T01:32:14.756 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7804679161807109) > weight (0.25) 2024-09-18T01:32:14.756 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.30717229392796097) > weight (0.25) 2024-09-18T01:32:14.756 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.4981049390932135) > weight (0.25) 2024-09-18T01:32:14.756 INFO:tasks.rbd_mirror_thrash:waiting for 96 secs before thrashing 2024-09-18T01:32:17.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:32:17.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:32:17.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:32:17.469 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_7 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_7 512M 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 512M ']' 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=512M 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:32:17.470 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_7 2024-09-18T01:32:17.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_7 2024-09-18T01:32:17.516 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:32:17.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:17.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:32:17.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:32:17.517 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2024-09-18T01:32:18.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:32:18.073 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2024-09-18T01:32:18.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_7': 2024-09-18T01:32:18.109 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 18e5c2d68dfd 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.18e5c2d68dfd 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:32:17 2024 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:32:17 2024 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:32:17 2024 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:32:18.110 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 9ec927aa-1f3f-40a1-92ec-0091b8e30713 2024-09-18T01:32:18.111 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:32:18.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_7 2024-09-18T01:32:18.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:32:18.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:18.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:32:18.113 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:32:18.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=11 2024-09-18T01:32:18.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:32:18.115 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:32:18.273 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 28558336 bytes 2024-09-18T01:32:18.414 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 96863232 bytes 2024-09-18T01:32:18.520 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 124047360 bytes 2024-09-18T01:32:18.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 136219648 bytes 2024-09-18T01:32:18.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 150916096 bytes 2024-09-18T01:32:18.834 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 184185856 bytes 2024-09-18T01:32:18.951 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 216785920 bytes 2024-09-18T01:32:19.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 279966720 bytes 2024-09-18T01:32:19.195 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 304922624 bytes 2024-09-18T01:32:19.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 348283904 bytes 2024-09-18T01:32:19.413 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 361161728 bytes 2024-09-18T01:32:19.519 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 385673216 bytes 2024-09-18T01:32:19.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 400932864 bytes 2024-09-18T01:32:19.712 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 417349632 bytes 2024-09-18T01:32:19.823 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 443355136 bytes 2024-09-18T01:32:19.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 481747968 bytes 2024-09-18T01:32:20.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 512398336 bytes 2024-09-18T01:32:20.187 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 557878272 bytes 2024-09-18T01:32:20.311 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 605359104 bytes 2024-09-18T01:32:20.465 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 678194176 bytes 2024-09-18T01:32:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 775364608 bytes 2024-09-18T01:32:20.765 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 813417472 bytes 2024-09-18T01:32:20.852 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 815481856 bytes 2024-09-18T01:32:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 897209344 bytes 2024-09-18T01:32:21.159 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 954488832 bytes 2024-09-18T01:32:21.260 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 971321344 bytes 2024-09-18T01:32:21.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1015878656 bytes 2024-09-18T01:32:21.472 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1018249216 bytes 2024-09-18T01:32:21.614 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1084782592 bytes 2024-09-18T01:32:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1128748032 bytes 2024-09-18T01:32:21.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1183395840 bytes 2024-09-18T01:32:22.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1237517312 bytes 2024-09-18T01:32:22.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1258071040 bytes 2024-09-18T01:32:22.238 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1300112384 bytes 2024-09-18T01:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1368020992 bytes 2024-09-18T01:32:22.464 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1368639488 bytes 2024-09-18T01:32:22.574 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1396648960 bytes 2024-09-18T01:32:22.690 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1425410048 bytes 2024-09-18T01:32:22.810 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1460452352 bytes 2024-09-18T01:32:22.891 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1461086208 bytes 2024-09-18T01:32:22.985 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1461691392 bytes 2024-09-18T01:32:23.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1479150592 bytes 2024-09-18T01:32:23.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1565708288 bytes 2024-09-18T01:32:23.360 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1582617600 bytes 2024-09-18T01:32:23.462 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1605030912 bytes 2024-09-18T01:32:23.646 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1720349696 bytes 2024-09-18T01:32:23.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1771392000 bytes 2024-09-18T01:32:23.883 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1777783808 bytes 2024-09-18T01:32:23.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1778419712 bytes 2024-09-18T01:32:24.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 1779046400 bytes 2024-09-18T01:32:24.175 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 1827064832 bytes 2024-09-18T01:32:24.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 1860039680 bytes 2024-09-18T01:32:24.397 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 1883302912 bytes 2024-09-18T01:32:24.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 1948044288 bytes 2024-09-18T01:32:24.668 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 1988583424 bytes 2024-09-18T01:32:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2005976064 bytes 2024-09-18T01:32:24.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2058288128 bytes 2024-09-18T01:32:25.030 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2103728128 bytes 2024-09-18T01:32:25.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2135955456 bytes 2024-09-18T01:32:25.236 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2136567808 bytes 2024-09-18T01:32:25.350 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2166978560 bytes 2024-09-18T01:32:25.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2167565312 bytes 2024-09-18T01:32:25.587 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2239334400 bytes 2024-09-18T01:32:25.691 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2262530048 bytes 2024-09-18T01:32:25.824 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2310943744 bytes 2024-09-18T01:32:25.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2365806592 bytes 2024-09-18T01:32:26.067 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2383367168 bytes 2024-09-18T01:32:26.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2447695872 bytes 2024-09-18T01:32:26.307 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2470767616 bytes 2024-09-18T01:32:26.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2493942784 bytes 2024-09-18T01:32:26.557 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2540443648 bytes 2024-09-18T01:32:26.667 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2569672704 bytes 2024-09-18T01:32:26.856 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 2681778176 bytes 2024-09-18T01:32:26.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 2725079040 bytes 2024-09-18T01:32:27.150 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 2808707072 bytes 2024-09-18T01:32:27.252 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 2829375488 bytes 2024-09-18T01:32:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 2908716032 bytes 2024-09-18T01:32:27.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 2909996032 bytes 2024-09-18T01:32:27.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3042152448 bytes 2024-09-18T01:32:27.793 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3062512640 bytes 2024-09-18T01:32:27.945 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3138720768 bytes 2024-09-18T01:32:28.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3188799488 bytes 2024-09-18T01:32:28.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3280591872 bytes 2024-09-18T01:32:28.352 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3303581696 bytes 2024-09-18T01:32:28.448 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3318821888 bytes 2024-09-18T01:32:28.547 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3338721280 bytes 2024-09-18T01:32:28.664 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3369863168 bytes 2024-09-18T01:32:28.781 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3396421632 bytes 2024-09-18T01:32:28.958 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3494398976 bytes 2024-09-18T01:32:29.079 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3537802240 bytes 2024-09-18T01:32:29.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:32:29.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:32:29.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:32:29.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:32:29.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_7 present 2024-09-18T01:32:29.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:32:29.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:29.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:32:29.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=present 2024-09-18T01:32:29.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:32:29.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:32:29.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:32:29.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_7 2024-09-18T01:32:29.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:32:29.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:32:29.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_7 2024-09-18T01:32:29.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-18T01:32:29.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:32:29.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id=11c37eb4e8be 2024-09-18T01:32:29.164 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:32:29.165 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:32:29.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_7 present 11c37eb4e8be 2024-09-18T01:32:29.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:32:29.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:29.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:32:29.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=present 2024-09-18T01:32:29.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id=11c37eb4e8be 2024-09-18T01:32:29.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:32:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:32:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_7 2024-09-18T01:32:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:32:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:32:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_7 2024-09-18T01:32:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-18T01:32:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:32:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id=11c37eb4e8be 2024-09-18T01:32:29.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 11c37eb4e8be 2024-09-18T01:32:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -z 11c37eb4e8be -o 11c37eb4e8be = 11c37eb4e8be 2024-09-18T01:32:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=present 2024-09-18T01:32:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test present = present 2024-09-18T01:32:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:32:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ disable_mirror cluster2 mirror image_7 2024-09-18T01:32:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:32:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:32:29.310 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_7 2024-09-18T01:32:30.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:32:30.086+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5a31000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=191 2024-09-18T01:32:30.087 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:32:30.086+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5a31000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=191, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b28075f7-24f5-441c-873b-85d148cb477b, primary_snap_id=bf, last_copied_object_number=128, snap_seqs={191=18446744073709551614}] 2024-09-18T01:32:35.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring disabled 2024-09-18T01:32:35.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2024-09-18T01:32:35.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:32:35.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:35.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:32:35.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:32:35.101 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:32:35.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:32:35.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:32:35.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_7 2024-09-18T01:32:35.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:32:35.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:32:35.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_7 2024-09-18T01:32:35.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-18T01:32:35.102 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:32:35.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-18T01:32:35.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:32:35.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:32:35.143 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:32:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2024-09-18T01:32:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:32:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:32:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:32:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:32:35.245 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:32:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:32:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_7 2024-09-18T01:32:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:32:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:32:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_7 2024-09-18T01:32:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-09-18T01:32:35.246 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:32:35.279 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-09-18T01:32:35.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:32:35.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:32:35.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:32:35.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:32:35.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ purge_snapshots cluster2 mirror image_7 2024-09-18T01:32:35.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:32:35.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:35.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:32:35.284 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap purge mirror/image_7 2024-09-18T01:32:35.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_7 2024-09-18T01:32:35.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:32:35.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:35.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:32:35.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:32:35.329 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:32:35.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_7 2024-09-18T01:32:35.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:32:35.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:35.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_7 2024-09-18T01:32:35.330 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-09-18T01:32:35.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-09-18T01:32:59.471 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_8 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_8 512M 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:32:59.475 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 512M ']' 2024-09-18T01:32:59.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=512M 2024-09-18T01:32:59.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:32:59.476 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_8 2024-09-18T01:32:59.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_8 2024-09-18T01:32:59.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:32:59.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:32:59.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:32:59.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:32:59.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2024-09-18T01:33:00.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:33:00.083 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2024-09-18T01:33:00.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_8': 2024-09-18T01:33:00.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:33:00.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:33:00.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:33:00.119 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 18f630bb8abb 2024-09-18T01:33:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.18f630bb8abb 2024-09-18T01:33:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:33:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:33:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:33:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:33:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:32:59 2024 2024-09-18T01:33:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:32:59 2024 2024-09-18T01:33:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:32:59 2024 2024-09-18T01:33:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:33:00.120 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:33:00.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: eac20ea4-3ebe-40a0-bc61-c04041d99675 2024-09-18T01:33:00.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:33:00.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_8 2024-09-18T01:33:00.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:00.122 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:00.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:33:00.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:33:00.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=11 2024-09-18T01:33:00.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:33:00.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:33:00.293 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 36209664 bytes 2024-09-18T01:33:00.373 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 36834304 bytes 2024-09-18T01:33:00.501 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 89744384 bytes 2024-09-18T01:33:00.605 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 114203648 bytes 2024-09-18T01:33:00.718 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 146367488 bytes 2024-09-18T01:33:00.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 270483456 bytes 2024-09-18T01:33:01.016 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 295628800 bytes 2024-09-18T01:33:01.162 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 370899968 bytes 2024-09-18T01:33:01.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 387086336 bytes 2024-09-18T01:33:01.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 444454912 bytes 2024-09-18T01:33:01.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 487924736 bytes 2024-09-18T01:33:01.647 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 532752384 bytes 2024-09-18T01:33:01.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 570246144 bytes 2024-09-18T01:33:01.877 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 593952768 bytes 2024-09-18T01:33:01.973 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 608958464 bytes 2024-09-18T01:33:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 632438784 bytes 2024-09-18T01:33:02.103 INFO:tasks.ceph.cluster2.osd.2.smithi150.stderr:2024-09-18T01:33:02.100+0000 7f80d5d16640 -1 reset not still connected to 0x5641831faf70 2024-09-18T01:33:02.202 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 677868544 bytes 2024-09-18T01:33:02.348 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 737723392 bytes 2024-09-18T01:33:02.477 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 787047424 bytes 2024-09-18T01:33:02.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 830831616 bytes 2024-09-18T01:33:02.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 848220160 bytes 2024-09-18T01:33:02.808 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 881030144 bytes 2024-09-18T01:33:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 912754688 bytes 2024-09-18T01:33:03.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 938636288 bytes 2024-09-18T01:33:03.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 970428416 bytes 2024-09-18T01:33:03.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1009521664 bytes 2024-09-18T01:33:03.388 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1050380288 bytes 2024-09-18T01:33:03.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1068272640 bytes 2024-09-18T01:33:03.634 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1130234880 bytes 2024-09-18T01:33:03.751 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1164598272 bytes 2024-09-18T01:33:03.866 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1201828864 bytes 2024-09-18T01:33:04.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1284082688 bytes 2024-09-18T01:33:04.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1345295360 bytes 2024-09-18T01:33:04.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1394520064 bytes 2024-09-18T01:33:04.456 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1492563968 bytes 2024-09-18T01:33:04.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1544696832 bytes 2024-09-18T01:33:04.674 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1545316352 bytes 2024-09-18T01:33:04.812 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1604847616 bytes 2024-09-18T01:33:04.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1625121792 bytes 2024-09-18T01:33:05.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1649741824 bytes 2024-09-18T01:33:05.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1679795200 bytes 2024-09-18T01:33:05.263 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1730455552 bytes 2024-09-18T01:33:05.400 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1781621760 bytes 2024-09-18T01:33:05.512 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1817395200 bytes 2024-09-18T01:33:05.618 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1836364800 bytes 2024-09-18T01:33:05.775 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1932888064 bytes 2024-09-18T01:33:05.927 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1984640000 bytes 2024-09-18T01:33:06.043 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 2024685568 bytes 2024-09-18T01:33:06.186 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 2084397056 bytes 2024-09-18T01:33:06.317 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2133926912 bytes 2024-09-18T01:33:06.422 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2151632896 bytes 2024-09-18T01:33:06.544 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2200807424 bytes 2024-09-18T01:33:06.732 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2320555008 bytes 2024-09-18T01:33:06.896 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2402546688 bytes 2024-09-18T01:33:07.013 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2435279872 bytes 2024-09-18T01:33:07.138 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2478546944 bytes 2024-09-18T01:33:07.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2582662144 bytes 2024-09-18T01:33:07.463 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2649159680 bytes 2024-09-18T01:33:07.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2659546112 bytes 2024-09-18T01:33:07.709 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2744707072 bytes 2024-09-18T01:33:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2803675136 bytes 2024-09-18T01:33:07.997 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2870834176 bytes 2024-09-18T01:33:08.121 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2921527296 bytes 2024-09-18T01:33:08.277 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2985003008 bytes 2024-09-18T01:33:08.366 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2992354304 bytes 2024-09-18T01:33:08.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 3042693120 bytes 2024-09-18T01:33:08.611 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 3089666048 bytes 2024-09-18T01:33:08.730 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 3106895872 bytes 2024-09-18T01:33:08.879 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 3177267200 bytes 2024-09-18T01:33:08.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 3218396160 bytes 2024-09-18T01:33:09.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 3274842112 bytes 2024-09-18T01:33:09.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 3326344192 bytes 2024-09-18T01:33:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 3366978560 bytes 2024-09-18T01:33:09.523 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 3425830912 bytes 2024-09-18T01:33:09.651 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 3472627712 bytes 2024-09-18T01:33:09.742 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3475733504 bytes 2024-09-18T01:33:09.853 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3508754432 bytes 2024-09-18T01:33:09.996 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3571579904 bytes 2024-09-18T01:33:10.133 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3638316032 bytes 2024-09-18T01:33:10.244 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3661480960 bytes 2024-09-18T01:33:10.338 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3670573056 bytes 2024-09-18T01:33:10.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3699161088 bytes 2024-09-18T01:33:10.609 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3775707136 bytes 2024-09-18T01:33:10.729 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3830731776 bytes 2024-09-18T01:33:10.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3858821120 bytes 2024-09-18T01:33:10.981 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3908692992 bytes 2024-09-18T01:33:11.103 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3945680896 bytes 2024-09-18T01:33:11.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:33:11.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:33:11.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:33:11.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:33:11.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_8 present 2024-09-18T01:33:11.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:33:11.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:11.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:33:11.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=present 2024-09-18T01:33:11.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:33:11.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:33:11.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:33:11.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_8 2024-09-18T01:33:11.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:33:11.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:33:11.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_8 2024-09-18T01:33:11.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-18T01:33:11.131 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:33:11.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id=11c3fcd5c9f8 2024-09-18T01:33:11.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:33:11.173 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:33:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_8 present 11c3fcd5c9f8 2024-09-18T01:33:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=present 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id=11c3fcd5c9f8 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_8 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_8 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-18T01:33:11.275 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:33:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id=11c3fcd5c9f8 2024-09-18T01:33:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 11c3fcd5c9f8 2024-09-18T01:33:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -z 11c3fcd5c9f8 -o 11c3fcd5c9f8 = 11c3fcd5c9f8 2024-09-18T01:33:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=present 2024-09-18T01:33:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test present = present 2024-09-18T01:33:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:33:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ disable_mirror cluster2 mirror image_8 2024-09-18T01:33:11.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:11.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:11.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:33:11.319 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_8 2024-09-18T01:33:12.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:33:12.086+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5593c00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=192 2024-09-18T01:33:12.088 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:33:12.086+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5593c00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=192, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b28075f7-24f5-441c-873b-85d148cb477b, primary_snap_id=c0, last_copied_object_number=128, snap_seqs={192=18446744073709551614}] 2024-09-18T01:33:12.592 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.3371423970504801) > weight (0.25) 2024-09-18T01:33:12.592 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.842344651048702) > weight (0.25) 2024-09-18T01:33:12.592 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-09-18T01:33:12.593 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-09-18T01:33:12.593 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-09-18T01:33:12.593 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:33:12.593+0000 7faa195c8640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 68627) UID: 1000 2024-09-18T01:33:14.594 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-09-18T01:33:14.594 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:33:14.594 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-09-18T01:33:14.595 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-09-18T01:33:14.595 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-09-18T01:33:14.598 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-09-18T01:33:14.598 INFO:tasks.rbd_mirror_thrash:waiting for 87 secs before thrashing 2024-09-18T01:33:17.090 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring disabled 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_8 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_8 2024-09-18T01:33:17.099 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-18T01:33:17.100 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:33:17.340 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-18T01:33:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:33:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:33:17.344 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:33:17.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2024-09-18T01:33:17.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:33:17.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:17.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:33:17.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:33:17.446 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:33:17.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:33:17.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:33:17.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_8 2024-09-18T01:33:17.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:33:17.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:33:17.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_8 2024-09-18T01:33:17.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-09-18T01:33:17.447 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:33:17.484 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-09-18T01:33:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:33:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:33:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:33:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:33:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ purge_snapshots cluster2 mirror image_8 2024-09-18T01:33:17.488 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:33:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap purge mirror/image_8 2024-09-18T01:33:17.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_8 2024-09-18T01:33:17.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:17.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:17.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:33:17.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:33:17.536 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:33:17.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_8 2024-09-18T01:33:17.537 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:17.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:17.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_8 2024-09-18T01:33:17.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-09-18T01:33:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-09-18T01:33:41.529 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:33:41.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:33:41.532 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_9 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_9 512M 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 512M ']' 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=512M 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:33:41.533 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_9 2024-09-18T01:33:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_9 2024-09-18T01:33:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:41.580 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:33:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:33:41.581 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2024-09-18T01:33:42.130 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:33:42.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2024-09-18T01:33:42.177 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_9': 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 190d15859b5f 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.190d15859b5f 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:33:41 2024 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:33:41 2024 2024-09-18T01:33:42.178 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:33:41 2024 2024-09-18T01:33:42.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:33:42.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:33:42.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 890632f3-5dcb-4356-8c29-dd62bf0f39bc 2024-09-18T01:33:42.179 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:33:42.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_9 2024-09-18T01:33:42.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:42.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:42.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:33:42.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:33:42.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=5 2024-09-18T01:33:42.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:33:42.183 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 46789632 bytes 2024-09-18T01:33:42.442 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 57219072 bytes 2024-09-18T01:33:42.624 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 168649728 bytes 2024-09-18T01:33:42.746 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 210363392 bytes 2024-09-18T01:33:42.854 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 242589696 bytes 2024-09-18T01:33:42.965 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 262649856 bytes 2024-09-18T01:33:43.089 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 303966208 bytes 2024-09-18T01:33:43.198 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 327201792 bytes 2024-09-18T01:33:43.288 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 336691200 bytes 2024-09-18T01:33:43.399 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 363254784 bytes 2024-09-18T01:33:43.548 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 438099968 bytes 2024-09-18T01:33:43.698 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 512286720 bytes 2024-09-18T01:33:43.801 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 548625408 bytes 2024-09-18T01:33:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 577276928 bytes 2024-09-18T01:33:44.074 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 651941888 bytes 2024-09-18T01:33:44.213 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 714586112 bytes 2024-09-18T01:33:44.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 735116288 bytes 2024-09-18T01:33:44.428 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 763616256 bytes 2024-09-18T01:33:44.538 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 784137216 bytes 2024-09-18T01:33:44.723 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 895457280 bytes 2024-09-18T01:33:44.913 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 955426816 bytes 2024-09-18T01:33:45.056 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 1018310656 bytes 2024-09-18T01:33:45.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 1059315712 bytes 2024-09-18T01:33:45.318 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 1123287040 bytes 2024-09-18T01:33:45.445 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 1174660096 bytes 2024-09-18T01:33:45.549 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1193418752 bytes 2024-09-18T01:33:45.693 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1247823872 bytes 2024-09-18T01:33:45.830 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1328909312 bytes 2024-09-18T01:33:46.039 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1434820608 bytes 2024-09-18T01:33:46.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1444550656 bytes 2024-09-18T01:33:46.280 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1505187840 bytes 2024-09-18T01:33:46.367 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1511930880 bytes 2024-09-18T01:33:46.452 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1512546304 bytes 2024-09-18T01:33:46.556 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1536673792 bytes 2024-09-18T01:33:46.677 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1579307008 bytes 2024-09-18T01:33:46.787 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1613173760 bytes 2024-09-18T01:33:46.881 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1622279168 bytes 2024-09-18T01:33:46.993 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1659432960 bytes 2024-09-18T01:33:47.135 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1705249792 bytes 2024-09-18T01:33:47.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:33:47.188 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_9 present 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=present 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_9 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:33:47.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:33:47.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_9 2024-09-18T01:33:47.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-18T01:33:47.190 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:33:47.231 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id=11c3ce8c6791 2024-09-18T01:33:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:33:47.232 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:33:47.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_9 present 11c3ce8c6791 2024-09-18T01:33:47.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:33:47.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:47.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:33:47.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=present 2024-09-18T01:33:47.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id=11c3ce8c6791 2024-09-18T01:33:47.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:33:47.334 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:33:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_9 2024-09-18T01:33:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:33:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:33:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_9 2024-09-18T01:33:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-18T01:33:47.335 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:33:47.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id=11c3ce8c6791 2024-09-18T01:33:47.377 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 11c3ce8c6791 2024-09-18T01:33:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -z 11c3ce8c6791 -o 11c3ce8c6791 = 11c3ce8c6791 2024-09-18T01:33:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=present 2024-09-18T01:33:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test present = present 2024-09-18T01:33:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:33:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ disable_mirror cluster2 mirror image_9 2024-09-18T01:33:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:33:47.378 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_9 2024-09-18T01:33:48.140 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:33:48.138+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef56cd800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=193 2024-09-18T01:33:48.140 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:33:48.138+0000 7f364068d640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef56cd800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=194, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b28075f7-24f5-441c-873b-85d148cb477b, primary_snap_id=c1, last_copied_object_number=128, snap_seqs={193=18446744073709551614}] 2024-09-18T01:33:50.481 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:33:50.479+0000 7ffa516db640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ccedcec820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:33:50.481 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:33:50.479+0000 7ffa586e9640 -1 rbd::mirror::ImageReplayer: 0x55ccee32a780 [3/890632f3-5dcb-4356-8c29-dd62bf0f39bc] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:33:50.754 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:33:50.752+0000 7ffa51edc640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55ccedad1a00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-09-18T01:33:50.754 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:33:50.752+0000 7ffa57ee8640 -1 rbd::mirror::ImageReplayer: 0x55ccee32a780 [3/890632f3-5dcb-4356-8c29-dd62bf0f39bc] operator(): start failed: (85) Interrupted system call should be restarted 2024-09-18T01:33:50.757 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7909627154010639) > weight (0.25) 2024-09-18T01:33:50.757 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7296537131201244) > weight (0.25) 2024-09-18T01:33:50.757 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-09-18T01:33:50.758 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-09-18T01:33:50.758 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-18T01:33:50.758 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-09-18T01:33:50.758 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:33:50.759 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:33:50.757+0000 7f8e7ed0f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 68347) UID: 1000 2024-09-18T01:33:53.142 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring disabled 2024-09-18T01:33:53.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2024-09-18T01:33:53.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:33:53.152 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:33:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:33:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:33:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:33:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:33:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_9 2024-09-18T01:33:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:33:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:33:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_9 2024-09-18T01:33:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-18T01:33:53.153 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:33:53.189 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-18T01:33:53.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:33:53.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:33:53.193 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:33:53.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2024-09-18T01:33:53.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:33:53.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:53.295 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:33:53.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:33:53.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:33:53.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:33:53.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:33:53.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_9 2024-09-18T01:33:53.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:33:53.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:33:53.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_9 2024-09-18T01:33:53.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-09-18T01:33:53.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:33:53.332 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-09-18T01:33:53.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:33:53.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:33:53.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:33:53.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:33:53.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ purge_snapshots cluster2 mirror image_9 2024-09-18T01:33:53.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:53.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:53.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:33:53.336 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap purge mirror/image_9 2024-09-18T01:33:53.381 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_9 2024-09-18T01:33:53.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:53.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:53.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:33:53.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:33:53.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:33:53.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_9 2024-09-18T01:33:53.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:33:53.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:33:53.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_9 2024-09-18T01:33:53.383 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-09-18T01:33:53.427 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-09-18T01:33:56.860 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2024-09-18T01:33:56.861 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-09-18T01:33:56.861 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-09-18T01:33:56.865 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-09-18T01:33:56.865 INFO:tasks.rbd_mirror_thrash:waiting for 73 secs before thrashing 2024-09-18T01:34:16.499 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:34:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:34:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for i in `seq 1 10` 2024-09-18T01:34:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image=image_10 2024-09-18T01:34:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ create_image cluster2 mirror image_10 512M 2024-09-18T01:34:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:34:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:34:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:34:16.503 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:34:16.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:34:16.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local size=128 2024-09-18T01:34:16.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -n 512M ']' 2024-09-18T01:34:16.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ size=512M 2024-09-18T01:34:16.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ shift 2024-09-18T01:34:16.504 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock mirror/image_10 2024-09-18T01:34:16.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ enable_mirror cluster2 mirror image_10 2024-09-18T01:34:16.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:34:16.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:16.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:34:16.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local mode=snapshot 2024-09-18T01:34:16.551 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2024-09-18T01:34:17.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring enabled 2024-09-18T01:34:17.086 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2024-09-18T01:34:17.123 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:rbd image 'image_10': 2024-09-18T01:34:17.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: size 512 MiB in 128 objects 2024-09-18T01:34:17.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: order 22 (4 MiB objects) 2024-09-18T01:34:17.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: snapshot_count: 1 2024-09-18T01:34:17.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: id: 192c539c925 2024-09-18T01:34:17.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: block_name_prefix: rbd_data.192c539c925 2024-09-18T01:34:17.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: format: 2 2024-09-18T01:34:17.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: features: layering, exclusive-lock 2024-09-18T01:34:17.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: op_features: 2024-09-18T01:34:17.124 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: flags: 2024-09-18T01:34:17.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: create_timestamp: Wed Sep 18 01:34:16 2024 2024-09-18T01:34:17.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: access_timestamp: Wed Sep 18 01:34:16 2024 2024-09-18T01:34:17.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: modify_timestamp: Wed Sep 18 01:34:16 2024 2024-09-18T01:34:17.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring state: enabled 2024-09-18T01:34:17.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring mode: snapshot 2024-09-18T01:34:17.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring global id: 3570721b-dd2c-44af-9f74-3f49e0dc75dd 2024-09-18T01:34:17.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout: mirroring primary: true 2024-09-18T01:34:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stress_write_image cluster2 mirror image_10 2024-09-18T01:34:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:34:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:34:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-09-18T01:34:17.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local duration=13 2024-09-18T01:34:17.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:34:17.129 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2024-09-18T01:34:17.297 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:100 IOs, 41100288 bytes 2024-09-18T01:34:17.453 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:200 IOs, 135720960 bytes 2024-09-18T01:34:17.565 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:300 IOs, 161250304 bytes 2024-09-18T01:34:17.675 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:400 IOs, 188113920 bytes 2024-09-18T01:34:17.784 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:500 IOs, 219894784 bytes 2024-09-18T01:34:17.884 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:600 IOs, 242042880 bytes 2024-09-18T01:34:18.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:700 IOs, 284381184 bytes 2024-09-18T01:34:18.097 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:800 IOs, 305294336 bytes 2024-09-18T01:34:18.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:900 IOs, 326088704 bytes 2024-09-18T01:34:18.299 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1000 IOs, 333784064 bytes 2024-09-18T01:34:18.415 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1100 IOs, 369951744 bytes 2024-09-18T01:34:18.576 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1200 IOs, 464293888 bytes 2024-09-18T01:34:18.686 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1300 IOs, 488790016 bytes 2024-09-18T01:34:18.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1400 IOs, 593432576 bytes 2024-09-18T01:34:18.947 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1500 IOs, 598261760 bytes 2024-09-18T01:34:19.058 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1600 IOs, 629367808 bytes 2024-09-18T01:34:19.206 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1700 IOs, 688980992 bytes 2024-09-18T01:34:19.321 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1800 IOs, 725819392 bytes 2024-09-18T01:34:19.467 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:1900 IOs, 800266240 bytes 2024-09-18T01:34:19.595 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2000 IOs, 838486016 bytes 2024-09-18T01:34:19.699 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2100 IOs, 860679168 bytes 2024-09-18T01:34:19.850 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2200 IOs, 927764480 bytes 2024-09-18T01:34:19.960 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2300 IOs, 941417472 bytes 2024-09-18T01:34:20.051 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2400 IOs, 945167360 bytes 2024-09-18T01:34:20.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2500 IOs, 977067008 bytes 2024-09-18T01:34:20.314 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2600 IOs, 1037082624 bytes 2024-09-18T01:34:20.432 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2700 IOs, 1068403712 bytes 2024-09-18T01:34:20.554 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2800 IOs, 1108677632 bytes 2024-09-18T01:34:20.657 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:2900 IOs, 1132796928 bytes 2024-09-18T01:34:20.772 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3000 IOs, 1161320448 bytes 2024-09-18T01:34:20.901 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3100 IOs, 1206630400 bytes 2024-09-18T01:34:21.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3200 IOs, 1277109248 bytes 2024-09-18T01:34:21.147 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3300 IOs, 1281721344 bytes 2024-09-18T01:34:21.292 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3400 IOs, 1347064832 bytes 2024-09-18T01:34:21.401 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3500 IOs, 1373757440 bytes 2024-09-18T01:34:21.541 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3600 IOs, 1457948672 bytes 2024-09-18T01:34:21.679 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3700 IOs, 1495714816 bytes 2024-09-18T01:34:21.785 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3800 IOs, 1517934592 bytes 2024-09-18T01:34:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:3900 IOs, 1558838272 bytes 2024-09-18T01:34:22.022 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4000 IOs, 1595341824 bytes 2024-09-18T01:34:22.146 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4100 IOs, 1637011456 bytes 2024-09-18T01:34:22.300 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4200 IOs, 1707632640 bytes 2024-09-18T01:34:22.455 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4300 IOs, 1786002432 bytes 2024-09-18T01:34:22.575 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4400 IOs, 1820943360 bytes 2024-09-18T01:34:22.701 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4500 IOs, 1861014528 bytes 2024-09-18T01:34:22.790 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4600 IOs, 1868212224 bytes 2024-09-18T01:34:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4700 IOs, 1895325696 bytes 2024-09-18T01:34:23.033 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4800 IOs, 1953771520 bytes 2024-09-18T01:34:23.149 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:4900 IOs, 1982726144 bytes 2024-09-18T01:34:23.286 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5000 IOs, 2037788672 bytes 2024-09-18T01:34:23.420 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5100 IOs, 2092563456 bytes 2024-09-18T01:34:23.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5200 IOs, 2159907840 bytes 2024-09-18T01:34:23.666 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5300 IOs, 2174897152 bytes 2024-09-18T01:34:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5400 IOs, 2220029952 bytes 2024-09-18T01:34:23.897 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5500 IOs, 2244746240 bytes 2024-09-18T01:34:24.014 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5600 IOs, 2278944768 bytes 2024-09-18T01:34:24.148 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5700 IOs, 2327447552 bytes 2024-09-18T01:34:24.241 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5800 IOs, 2349728768 bytes 2024-09-18T01:34:24.393 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:5900 IOs, 2411333632 bytes 2024-09-18T01:34:24.524 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6000 IOs, 2454241280 bytes 2024-09-18T01:34:24.633 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6100 IOs, 2476459008 bytes 2024-09-18T01:34:24.745 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6200 IOs, 2500546560 bytes 2024-09-18T01:34:24.869 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6300 IOs, 2554869760 bytes 2024-09-18T01:34:25.020 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6400 IOs, 2615419904 bytes 2024-09-18T01:34:25.141 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6500 IOs, 2655286272 bytes 2024-09-18T01:34:25.257 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6600 IOs, 2692096000 bytes 2024-09-18T01:34:25.395 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6700 IOs, 2747416576 bytes 2024-09-18T01:34:25.494 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6800 IOs, 2763997184 bytes 2024-09-18T01:34:25.597 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:6900 IOs, 2785372160 bytes 2024-09-18T01:34:25.711 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7000 IOs, 2821125120 bytes 2024-09-18T01:34:25.832 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7100 IOs, 2862108672 bytes 2024-09-18T01:34:25.917 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7200 IOs, 2867550208 bytes 2024-09-18T01:34:26.057 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7300 IOs, 2937467904 bytes 2024-09-18T01:34:26.160 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7400 IOs, 2959577088 bytes 2024-09-18T01:34:26.283 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7500 IOs, 2987511808 bytes 2024-09-18T01:34:26.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7600 IOs, 3031249920 bytes 2024-09-18T01:34:26.498 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7700 IOs, 3033675776 bytes 2024-09-18T01:34:26.602 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7800 IOs, 3052668928 bytes 2024-09-18T01:34:26.773 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:7900 IOs, 3156981760 bytes 2024-09-18T01:34:26.899 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8000 IOs, 3181843456 bytes 2024-09-18T01:34:27.078 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8100 IOs, 3278885888 bytes 2024-09-18T01:34:27.197 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8200 IOs, 3333136384 bytes 2024-09-18T01:34:27.386 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8300 IOs, 3432617984 bytes 2024-09-18T01:34:27.491 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8400 IOs, 3467397120 bytes 2024-09-18T01:34:27.659 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8500 IOs, 3531952128 bytes 2024-09-18T01:34:27.788 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8600 IOs, 3584370688 bytes 2024-09-18T01:34:27.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8700 IOs, 3623750656 bytes 2024-09-18T01:34:28.045 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8800 IOs, 3690830848 bytes 2024-09-18T01:34:28.181 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:8900 IOs, 3727387648 bytes 2024-09-18T01:34:28.296 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9000 IOs, 3752367104 bytes 2024-09-18T01:34:28.444 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9100 IOs, 3822986240 bytes 2024-09-18T01:34:28.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9200 IOs, 3871412224 bytes 2024-09-18T01:34:28.670 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9300 IOs, 3887371264 bytes 2024-09-18T01:34:28.776 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9400 IOs, 3907260416 bytes 2024-09-18T01:34:28.912 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9500 IOs, 3964056576 bytes 2024-09-18T01:34:29.009 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9600 IOs, 3976384512 bytes 2024-09-18T01:34:29.139 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9700 IOs, 4021557248 bytes 2024-09-18T01:34:29.254 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9800 IOs, 4054319104 bytes 2024-09-18T01:34:29.382 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:9900 IOs, 4099668992 bytes 2024-09-18T01:34:29.543 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10000 IOs, 4190987264 bytes 2024-09-18T01:34:29.644 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10100 IOs, 4201904128 bytes 2024-09-18T01:34:29.743 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10200 IOs, 4217084928 bytes 2024-09-18T01:34:29.841 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10300 IOs, 4232527872 bytes 2024-09-18T01:34:29.963 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10400 IOs, 4271352832 bytes 2024-09-18T01:34:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:10500 IOs, 4288789504 bytes 2024-09-18T01:34:30.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=124 2024-09-18T01:34:30.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:34:30.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 124 -eq 124 ']' 2024-09-18T01:34:30.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:34:30.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_10 present 2024-09-18T01:34:30.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:34:30.136 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:34:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=present 2024-09-18T01:34:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:34:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:34:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:34:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_10 2024-09-18T01:34:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:34:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_10 2024-09-18T01:34:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-18T01:34:30.137 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:34:30.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id=11c3be19ab23 2024-09-18T01:34:30.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:34:30.180 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:34:30.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_10 present 11c3be19ab23 2024-09-18T01:34:30.281 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=present 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id=11c3be19ab23 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_10 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_10 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-18T01:34:30.282 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:34:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id=11c3be19ab23 2024-09-18T01:34:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 11c3be19ab23 2024-09-18T01:34:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -z 11c3be19ab23 -o 11c3be19ab23 = 11c3be19ab23 2024-09-18T01:34:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_state=present 2024-09-18T01:34:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test present = present 2024-09-18T01:34:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:34:30.327 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ disable_mirror cluster2 mirror image_10 2024-09-18T01:34:30.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:34:30.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:30.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:34:30.328 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_10 2024-09-18T01:34:31.115 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:34:31.114+0000 7f3640e8e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5a51400 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=194 2024-09-18T01:34:31.115 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:34:31.114+0000 7f3640e8e640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x557ef5a51400 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=195, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b28075f7-24f5-441c-873b-85d148cb477b, primary_snap_id=c2, last_copied_object_number=128, snap_seqs={194=18446744073709551614}] 2024-09-18T01:34:36.116 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:Mirroring disabled 2024-09-18T01:34:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2024-09-18T01:34:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:34:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:34:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local state=deleted 2024-09-18T01:34:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:34:36.125 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local s 2024-09-18T01:34:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:34:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_10 2024-09-18T01:34:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:34:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_10 2024-09-18T01:34:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-18T01:34:36.126 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:34:36.163 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-18T01:34:36.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ image_id= 2024-09-18T01:34:36.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-09-18T01:34:36.167 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0.1 2024-09-18T01:34:36.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2024-09-18T01:34:36.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:34:36.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:36.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:34:36.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local test_state=deleted 2024-09-18T01:34:36.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image_id= 2024-09-18T01:34:36.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_state=deleted 2024-09-18T01:34:36.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local current_image_id 2024-09-18T01:34:36.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ get_image_id cluster1 mirror image_10 2024-09-18T01:34:36.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:34:36.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:36.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local image=image_10 2024-09-18T01:34:36.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-09-18T01:34:36.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-09-18T01:34:36.304 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-09-18T01:34:36.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ current_image_id= 2024-09-18T01:34:36.308 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n '' 2024-09-18T01:34:36.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test deleted = deleted 2024-09-18T01:34:36.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:34:36.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ purge_snapshots cluster2 mirror image_10 2024-09-18T01:34:36.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:34:36.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:36.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:34:36.309 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster cluster2 snap purge mirror/image_10 2024-09-18T01:34:36.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image_retry cluster2 mirror image_10 2024-09-18T01:34:36.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:34:36.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:36.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:34:36.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 4 8 16 32 2024-09-18T01:34:36.354 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:34:36.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ remove_image cluster2 mirror image_10 2024-09-18T01:34:36.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:34:36.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:36.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local image=image_10 2024-09-18T01:34:36.355 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-09-18T01:34:36.398 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-09-18T01:34:41.599 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-09-18T01:34:41.599 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-09-18T01:34:41.600 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-09-18T01:34:41.600 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-09-18T01:34:41.600 INFO:teuthology.orchestra.run:waiting for 600 2024-09-18T01:34:41.601 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:34:41.599+0000 7fa8cd168640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 70228) UID: 1000 2024-09-18T01:34:47.702 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-09-18T01:34:47.702 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-09-18T01:34:47.702 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-09-18T01:34:47.705 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-09-18T01:34:47.706 INFO:tasks.rbd_mirror_thrash:waiting for 95 secs before thrashing 2024-09-18T01:34:59.564 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-09-18T01:34:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:34:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2024-09-18T01:34:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ tee -a /tmp/tmp.yeNdCEAq6i/rbd-mirror.test.log 2024-09-18T01:34:59.567 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ date '+%F %T' 2024-09-18T01:34:59.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo 2024-09-18 01:34:59 TEST: check if removed 'images'\''' OMAP are removed 2024-09-18T01:34:59.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:2024-09-18 01:34:59 TEST: check if removed images' OMAP are removed 2024-09-18T01:34:59.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_in_omap cluster1 mirror 2024-09-18T01:34:59.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:34:59.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:59.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2024-09-18T01:34:59.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:34:59.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:59.569 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local obj_name=rbd_mirroring 2024-09-18T01:34:59.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local filter=status_global 2024-09-18T01:34:59.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-18T01:34:59.570 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:34:59.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:34:59.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2024-09-18T01:34:59.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:34:59.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:59.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirroring 2024-09-18T01:34:59.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local filter=status_global 2024-09-18T01:34:59.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2024-09-18T01:34:59.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:34:59.571 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:59.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirroring 2024-09-18T01:34:59.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2024-09-18T01:34:59.572 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -c status_global 2024-09-18T01:34:59.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 0 = 0 2024-09-18T01:34:59.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=0 2024-09-18T01:34:59.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:34:59.892 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -eq 0 ']' 2024-09-18T01:34:59.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:34:59.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2024-09-18T01:34:59.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:34:59.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:59.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local obj_name=rbd_mirroring 2024-09-18T01:34:59.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local filter=image_ 2024-09-18T01:34:59.893 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-18T01:34:59.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:34:59.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:34:59.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2024-09-18T01:34:59.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:34:59.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:59.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirroring 2024-09-18T01:34:59.894 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local filter=image_ 2024-09-18T01:34:59.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2024-09-18T01:34:59.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:34:59.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:59.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirroring 2024-09-18T01:34:59.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -c image_ 2024-09-18T01:34:59.895 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2024-09-18T01:34:59.930 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 0 = 0 2024-09-18T01:34:59.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=0 2024-09-18T01:34:59.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:34:59.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -eq 0 ']' 2024-09-18T01:34:59.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:34:59.931 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2024-09-18T01:34:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:34:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local obj_name=rbd_mirror_leader 2024-09-18T01:34:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local filter=image_map 2024-09-18T01:34:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-18T01:34:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:34:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:34:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2024-09-18T01:34:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:34:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirror_leader 2024-09-18T01:34:59.933 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local filter=image_map 2024-09-18T01:34:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2024-09-18T01:34:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster1 2024-09-18T01:34:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirror_leader 2024-09-18T01:34:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -c image_map 2024-09-18T01:34:59.934 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2024-09-18T01:34:59.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 0 = 0 2024-09-18T01:34:59.966 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=0 2024-09-18T01:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -eq 0 ']' 2024-09-18T01:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_image_in_omap cluster2 mirror 2024-09-18T01:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2024-09-18T01:34:59.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:34:59.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:59.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local obj_name=rbd_mirroring 2024-09-18T01:34:59.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local filter=status_global 2024-09-18T01:34:59.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-18T01:34:59.968 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:34:59.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:34:59.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2024-09-18T01:34:59.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster2 2024-09-18T01:34:59.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:59.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirroring 2024-09-18T01:34:59.969 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local filter=status_global 2024-09-18T01:34:59.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2024-09-18T01:34:59.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster2 2024-09-18T01:34:59.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:34:59.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -c status_global 2024-09-18T01:34:59.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirroring 2024-09-18T01:34:59.970 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2024-09-18T01:34:59.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 0 = 0 2024-09-18T01:34:59.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=0 2024-09-18T01:34:59.998 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -eq 0 ']' 2024-09-18T01:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2024-09-18T01:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local obj_name=rbd_mirroring 2024-09-18T01:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local filter=image_ 2024-09-18T01:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-18T01:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:35:00.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:35:00.000 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2024-09-18T01:35:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster2 2024-09-18T01:35:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:35:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirroring 2024-09-18T01:35:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local filter=image_ 2024-09-18T01:35:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2024-09-18T01:35:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster2 2024-09-18T01:35:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:35:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirroring 2024-09-18T01:35:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2024-09-18T01:35:00.001 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -c image_ 2024-09-18T01:35:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 0 = 0 2024-09-18T01:35:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=0 2024-09-18T01:35:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:35:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -eq 0 ']' 2024-09-18T01:35:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:35:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2024-09-18T01:35:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:35:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local pool=mirror 2024-09-18T01:35:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local obj_name=rbd_mirror_leader 2024-09-18T01:35:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local filter=image_map 2024-09-18T01:35:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2024-09-18T01:35:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ sleep 0 2024-09-18T01:35:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:35:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2024-09-18T01:35:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster2 2024-09-18T01:35:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:35:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirror_leader 2024-09-18T01:35:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local filter=image_map 2024-09-18T01:35:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2024-09-18T01:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local cluster=cluster2 2024-09-18T01:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local pool=mirror 2024-09-18T01:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ local obj_name=rbd_mirror_leader 2024-09-18T01:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ grep -c image_map 2024-09-18T01:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2024-09-18T01:35:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test 0 = 0 2024-09-18T01:35:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ error_code=0 2024-09-18T01:35:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set -e 2024-09-18T01:35:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -eq 0 ']' 2024-09-18T01:35:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 0 2024-09-18T01:35:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ cleanup 0 2024-09-18T01:35:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local error_code=0 2024-09-18T01:35:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ set +e 2024-09-18T01:35:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -ne 0 ']' 2024-09-18T01:35:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -z '' ']' 2024-09-18T01:35:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster instance 2024-09-18T01:35:00.070 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T01:35:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2024-09-18T01:35:01.049 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:pool 'mirror' does not exist 2024-09-18T01:35:01.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T01:35:01.060 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2024-09-18T01:35:02.247 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:pool 'mirror' does not exist 2024-09-18T01:35:02.272 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T01:35:02.272 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2024-09-18T01:35:03.052 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:pool 'mirror_parent' does not exist 2024-09-18T01:35:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ CEPH_ARGS= 2024-09-18T01:35:03.064 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2024-09-18T01:35:03.838 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.837+0000 7f364dea8640 -1 rbd::mirror::LeaderWatcher: 0x557ef4875200 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2024-09-18T01:35:03.838 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.837+0000 7f3639e80640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x557ef578a480 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-18T01:35:03.838 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.837+0000 7f363967f640 -1 rbd::mirror::PoolWatcher: 0x557ef4ca4f00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:03.840 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.840+0000 7f364d6a7640 -1 rbd::mirror::LeaderWatcher: 0x557ef4875200 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-18T01:35:03.840 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.840+0000 7f364d6a7640 -1 rbd::mirror::LeaderWatcher: 0x557ef4875200 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2024-09-18T01:35:03.903 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.903+0000 7f364d6a7640 -1 rbd::mirror::LeaderWatcher: 0x557ef45d8c00 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2024-09-18T01:35:03.904 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.903+0000 7f364769b640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x557ef578a6c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-18T01:35:03.904 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.903+0000 7f364769b640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x557ef5b5ad80 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-18T01:35:03.904 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.904+0000 7f364769b640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x557ef5b5ab40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-18T01:35:03.904 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.904+0000 7f3640e8e640 -1 rbd::mirror::PoolWatcher: 0x557ef52b6960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:03.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.904+0000 7f3646e9a640 -1 rbd::mirror::PoolWatcher: 0x557ef4ca52c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:03.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.904+0000 7f364068d640 -1 rbd::mirror::PoolWatcher: 0x557ef58e4960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:03.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.904+0000 7f364068d640 -1 rbd::mirror::PoolWatcher: 0x557ef4c31e00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:03.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.904+0000 7f364769b640 -1 rbd::mirror::PoolWatcher: 0x557ef58e54a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:03.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.904+0000 7f364769b640 -1 rbd::mirror::PoolWatcher: 0x557ef58e4d20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:03.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.905+0000 7f364dea8640 -1 rbd::mirror::LeaderWatcher: 0x557ef45d8c00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-18T01:35:03.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:03.905+0000 7f364dea8640 -1 rbd::mirror::LeaderWatcher: 0x557ef45d8c00 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2024-09-18T01:35:04.249 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:pool 'mirror_parent' does not exist 2024-09-18T01:35:04.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2024-09-18T01:35:04.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stop_mirrors cluster1 2024-09-18T01:35:04.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1 2024-09-18T01:35:04.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local sig= 2024-09-18T01:35:04.261 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 0 3 2024-09-18T01:35:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:35:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stop_mirror cluster1:0 '' 2024-09-18T01:35:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1:0 2024-09-18T01:35:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local sig= 2024-09-18T01:35:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T01:35:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T01:35:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:35:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stop_mirror cluster1:1 '' 2024-09-18T01:35:04.264 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1:1 2024-09-18T01:35:04.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local sig= 2024-09-18T01:35:04.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T01:35:04.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T01:35:04.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:35:04.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stop_mirror cluster1:2 '' 2024-09-18T01:35:04.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1:2 2024-09-18T01:35:04.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local sig= 2024-09-18T01:35:04.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T01:35:04.265 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T01:35:04.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:35:04.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stop_mirror cluster1:3 '' 2024-09-18T01:35:04.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster1:3 2024-09-18T01:35:04.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local sig= 2024-09-18T01:35:04.266 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T01:35:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T01:35:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2024-09-18T01:35:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stop_mirrors cluster2 2024-09-18T01:35:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2 2024-09-18T01:35:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local sig= 2024-09-18T01:35:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:++ seq 0 3 2024-09-18T01:35:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:35:04.267 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stop_mirror cluster2:0 '' 2024-09-18T01:35:04.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2:0 2024-09-18T01:35:04.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local sig= 2024-09-18T01:35:04.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T01:35:04.269 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stop_mirror cluster2:1 '' 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2:1 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local sig= 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stop_mirror cluster2:2 '' 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2:2 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local sig= 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T01:35:04.270 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-09-18T01:35:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ stop_mirror cluster2:3 '' 2024-09-18T01:35:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local cluster=cluster2:3 2024-09-18T01:35:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ local sig= 2024-09-18T01:35:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test -n 1 2024-09-18T01:35:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ return 2024-09-18T01:35:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' -z 1 ']' 2024-09-18T01:35:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ test '' = /tmp/tmp.yeNdCEAq6i 2024-09-18T01:35:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ rm -Rf /tmp/tmp.yeNdCEAq6i 2024-09-18T01:35:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ '[' 0 -eq 0 ']' 2024-09-18T01:35:04.271 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ echo OK 2024-09-18T01:35:04.272 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stdout:OK 2024-09-18T01:35:04.272 INFO:tasks.workunit.cluster1.client.mirror.smithi150.stderr:+ exit 0 2024-09-18T01:35:04.273 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-18T01:35:04.273 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2024-09-18T01:35:04.398 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_stress.sh'] on cluster1.client.mirror... 2024-09-18T01:35:04.398 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2024-09-18T01:35:04.708 DEBUG:teuthology.parallel:result is None 2024-09-18T01:35:04.709 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-18T01:35:04.734 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-18T01:35:04.734 DEBUG:teuthology.orchestra.run.smithi150:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-09-18T01:35:04.799 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2024-09-18T01:35:04.799 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2024-09-18T01:35:04.810 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2024-09-18T01:35:04.811 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 14 2024-09-18T01:35:04.811 INFO:tasks.rbd_mirror_thrash:done joining 2024-09-18T01:35:04.811 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2024-09-18T01:35:04.849 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2024-09-18T01:35:04.849 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 16 2024-09-18T01:35:04.849 INFO:tasks.rbd_mirror_thrash:done joining 2024-09-18T01:35:04.849 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-18T01:35:04.883 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.3:waiting for process to exit 2024-09-18T01:35:04.883 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:04.884 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.883+0000 7ffa58eea640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 60148) UID: 1000 2024-09-18T01:35:04.884 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.884+0000 7ffa446c1640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55ccedcaa480 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-18T01:35:04.884 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.884+0000 7ffa586e9640 -1 rbd::mirror::InstanceWatcher: 0x55ccedd58380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5731 object: (2) No such file or directory 2024-09-18T01:35:04.885 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.884+0000 7ffa446c1640 -1 rbd::mirror::PoolWatcher: 0x55cced90a000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:04.885 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.884+0000 7ffa57ee8640 -1 rbd::mirror::InstanceWatcher: 0x55ccedd58700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5731 object: (2) No such file or directory 2024-09-18T01:35:04.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.884+0000 7ffa51edc640 -1 rbd::mirror::InstanceWatcher: 0x55ccedd58380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:04.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.884+0000 7ffa51edc640 -1 rbd::mirror::InstanceWatcher: 0x55ccedd58700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:04.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.884+0000 7ffa3d6b3640 -1 rbd::mirror::PoolWatcher: 0x55cced90af00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:04.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.884+0000 7ffa4aece640 -1 rbd::mirror::MirrorStatusUpdater 0x55ccedd9c5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.884+0000 7ffa4aece640 -1 rbd::mirror::MirrorStatusUpdater 0x55ccedd9c360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.884+0000 7ffa586e9640 -1 rbd::mirror::NamespaceReplayer: 0x55ccedd98000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.885+0000 7ffa586e9640 -1 rbd::mirror::NamespaceReplayer: 0x55ccedd981a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.885+0000 7ffa57ee8640 -1 rbd::mirror::LeaderWatcher: 0x55ccedf69200 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-18T01:35:04.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.885+0000 7ffa57ee8640 -1 rbd::mirror::LeaderWatcher: 0x55ccedf69200 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2024-09-18T01:35:04.887 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.885+0000 7ffa516db640 -1 rbd::mirror::MirrorStatusUpdater 0x55ccedd9c000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.888 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.885+0000 7ffa516db640 -1 rbd::mirror::MirrorStatusUpdater 0x55ccedd9c120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.888 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.885+0000 7ffa586e9640 -1 rbd::mirror::NamespaceReplayer: 0x55ccedd98000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.888 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.885+0000 7ffa586e9640 -1 rbd::mirror::LeaderWatcher: 0x55ccedf69200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:04.888 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.885+0000 7ffa586e9640 -1 rbd::mirror::NamespaceReplayer: 0x55ccedd981a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.888 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.885+0000 7ffa57ee8640 -1 rbd::mirror::InstanceWatcher: 0x55ccedd58a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5737 object: (2) No such file or directory 2024-09-18T01:35:04.888 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.885+0000 7ffa446c1640 -1 rbd::mirror::InstanceWatcher: 0x55ccedd58a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:04.888 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.886+0000 7ffa3deb4640 -1 rbd::mirror::MirrorStatusUpdater 0x55ccedfe46c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.888 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.886+0000 7ffa57ee8640 -1 rbd::mirror::NamespaceReplayer: 0x55ccee220000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.888 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.886+0000 7ffa43ec0640 -1 rbd::mirror::MirrorStatusUpdater 0x55ccedfab200 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.888 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.886+0000 7ffa586e9640 -1 rbd::mirror::NamespaceReplayer: 0x55ccee220000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.889 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.888+0000 7ffa57ee8640 -1 rbd::mirror::LeaderWatcher: 0x55ccedca6c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:04.889 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.888+0000 7ffa57ee8640 -1 rbd::mirror::InstanceWatcher: 0x55ccedd58000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5731 object: (2) No such file or directory 2024-09-18T01:35:04.889 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.888+0000 7ffa51edc640 -1 rbd::mirror::InstanceWatcher: 0x55ccedd58000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:04.890 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.889+0000 7ffa4b6cf640 -1 rbd::mirror::MirrorStatusUpdater 0x55ccedab90e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.890 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.889+0000 7ffa586e9640 -1 rbd::mirror::NamespaceReplayer: 0x55cced9996c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.890 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.889+0000 7ffa516db640 -1 rbd::mirror::MirrorStatusUpdater 0x55ccedab9680 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.890 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi150.stderr:2024-09-18T01:35:04.889+0000 7ffa57ee8640 -1 rbd::mirror::NamespaceReplayer: 0x55cced9996c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.901 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Stopped 2024-09-18T01:35:04.901 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-18T01:35:04.916 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.2:waiting for process to exit 2024-09-18T01:35:04.916 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:04.942 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.942+0000 7f059d320640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 71387) UID: 1000 2024-09-18T01:35:04.943 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.942+0000 7f059cb1f640 -1 rbd::mirror::LeaderWatcher: 0x5633ed919200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:04.944 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.943+0000 7f059c31e640 -1 rbd::mirror::InstanceWatcher: 0x5633ed71c380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6438 object: (2) No such file or directory 2024-09-18T01:35:04.944 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.943+0000 7f059cb1f640 -1 rbd::mirror::InstanceWatcher: 0x5633edbe8000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6438 object: (2) No such file or directory 2024-09-18T01:35:04.944 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.943+0000 7f059cb1f640 -1 rbd::mirror::InstanceWatcher: 0x5633ed71dc00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6441 object: (2) No such file or directory 2024-09-18T01:35:04.944 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.943+0000 7f0596312640 -1 rbd::mirror::InstanceWatcher: 0x5633ed71c380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:04.944 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.943+0000 7f0595b11640 -1 rbd::mirror::InstanceWatcher: 0x5633edbe8000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:04.944 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.943+0000 7f05882f6640 -1 rbd::mirror::InstanceWatcher: 0x5633ed71dc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:04.945 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.943+0000 7f058f304640 -1 rbd::mirror::MirrorStatusUpdater 0x5633ed77e5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.945 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.943+0000 7f058f304640 -1 rbd::mirror::MirrorStatusUpdater 0x5633ed77e360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.943+0000 7f059c31e640 -1 rbd::mirror::NamespaceReplayer: 0x5633ed77c000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.944+0000 7f0581ae9640 -1 rbd::mirror::MirrorStatusUpdater 0x5633ed9965a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.944+0000 7f059c31e640 -1 rbd::mirror::NamespaceReplayer: 0x5633ed77c1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.944+0000 7f059c31e640 -1 rbd::mirror::NamespaceReplayer: 0x5633ed3581a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.944+0000 7f0596312640 -1 rbd::mirror::MirrorStatusUpdater 0x5633ed77e000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.944+0000 7f0596312640 -1 rbd::mirror::MirrorStatusUpdater 0x5633ed77e120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.944+0000 7f059cb1f640 -1 rbd::mirror::NamespaceReplayer: 0x5633ed77c000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.944+0000 7f059cb1f640 -1 rbd::mirror::NamespaceReplayer: 0x5633ed77c1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.944+0000 7f0588af7640 -1 rbd::mirror::MirrorStatusUpdater 0x5633ed95d0e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.946 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.944+0000 7f059c31e640 -1 rbd::mirror::NamespaceReplayer: 0x5633ed3581a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.947 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.946+0000 7f059c31e640 -1 rbd::mirror::LeaderWatcher: 0x5633ed688c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:04.947 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.946+0000 7f059c31e640 -1 rbd::mirror::InstanceWatcher: 0x5633ed71c000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6438 object: (2) No such file or directory 2024-09-18T01:35:04.947 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.946+0000 7f0595b11640 -1 rbd::mirror::InstanceWatcher: 0x5633ed71c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:04.947 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.946+0000 7f058fb05640 -1 rbd::mirror::MirrorStatusUpdater 0x5633ed4867e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.947 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.946+0000 7f059c31e640 -1 rbd::mirror::NamespaceReplayer: 0x5633edf76000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.948 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.946+0000 7f0596312640 -1 rbd::mirror::MirrorStatusUpdater 0x5633ed4850e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:04.948 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi150.stderr:2024-09-18T01:35:04.946+0000 7f059cb1f640 -1 rbd::mirror::NamespaceReplayer: 0x5633edf76000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:04.958 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Stopped 2024-09-18T01:35:04.959 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-18T01:35:04.970 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.1:waiting for process to exit 2024-09-18T01:35:04.970 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:04.999 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:04.999+0000 7f68d1b18640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 63465) UID: 1000 2024-09-18T01:35:05.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:04.999+0000 7f68d1317640 -1 rbd::mirror::LeaderWatcher: 0x562b53a33200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:05.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:04.999+0000 7f68ca309640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x562b53780480 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-18T01:35:05.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:04.999+0000 7f68ca309640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x562b53780d80 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-18T01:35:05.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.000+0000 7f68cab0a640 -1 rbd::mirror::PoolWatcher: 0x562b53d750e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:05.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.000+0000 7f68ca309640 -1 rbd::mirror::PoolWatcher: 0x562b53e4c3c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:05.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.000+0000 7f68d0b16640 -1 rbd::mirror::InstanceWatcher: 0x562b53832700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6006 object: (2) No such file or directory 2024-09-18T01:35:05.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.000+0000 7f68c42fd640 -1 rbd::mirror::PoolWatcher: 0x562b53e4c960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:05.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.000+0000 7f68c3afc640 -1 rbd::mirror::PoolWatcher: 0x562b53e4de00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:05.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.000+0000 7f68bcaee640 -1 rbd::mirror::InstanceWatcher: 0x562b53832700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.000+0000 7f68d0b16640 -1 rbd::mirror::InstanceWatcher: 0x562b53832380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6004 object: (2) No such file or directory 2024-09-18T01:35:05.002 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.000+0000 7f68d1317640 -1 rbd::mirror::InstanceWatcher: 0x562b53832a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6004 object: (2) No such file or directory 2024-09-18T01:35:05.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68cab0a640 -1 rbd::mirror::InstanceWatcher: 0x562b53832380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68ca309640 -1 rbd::mirror::InstanceWatcher: 0x562b53832a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68b6ae2640 -1 rbd::mirror::MirrorStatusUpdater 0x562b53ab66c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68d0b16640 -1 rbd::mirror::NamespaceReplayer: 0x562b534aa1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68bd2ef640 -1 rbd::mirror::MirrorStatusUpdater 0x562b53a73200 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68d1317640 -1 rbd::mirror::NamespaceReplayer: 0x562b534aa1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68c42fd640 -1 rbd::mirror::MirrorStatusUpdater 0x562b5387a360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68d1317640 -1 rbd::mirror::NamespaceReplayer: 0x562b53878000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68c3afc640 -1 rbd::mirror::MirrorStatusUpdater 0x562b5387a5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.003 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68d0b16640 -1 rbd::mirror::NamespaceReplayer: 0x562b538781a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68cab0a640 -1 rbd::mirror::MirrorStatusUpdater 0x562b5387a000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68d1317640 -1 rbd::mirror::NamespaceReplayer: 0x562b53878000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68ca309640 -1 rbd::mirror::MirrorStatusUpdater 0x562b5387a120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.004 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.001+0000 7f68d1317640 -1 rbd::mirror::NamespaceReplayer: 0x562b538781a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.003+0000 7f68cab0a640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x562b53c5d440 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2024-09-18T01:35:05.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.003+0000 7f68ca309640 -1 rbd::mirror::PoolWatcher: 0x562b53e4d2c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:05.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.003+0000 7f68c42fd640 -1 rbd::mirror::PoolWatcher: 0x562b53e4c5a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2024-09-18T01:35:05.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.004+0000 7f68d0b16640 -1 rbd::mirror::LeaderWatcher: 0x562b5379ac00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2024-09-18T01:35:05.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.004+0000 7f68d0b16640 -1 rbd::mirror::LeaderWatcher: 0x562b5379ac00 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2024-09-18T01:35:05.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.004+0000 7f68d1317640 -1 rbd::mirror::LeaderWatcher: 0x562b5379ac00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:05.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.004+0000 7f68d1317640 -1 rbd::mirror::InstanceWatcher: 0x562b53832000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6004 object: (2) No such file or directory 2024-09-18T01:35:05.005 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.004+0000 7f68ca309640 -1 rbd::mirror::InstanceWatcher: 0x562b53832000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.005+0000 7f68c3afc640 -1 rbd::mirror::MirrorStatusUpdater 0x562b53794480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.005+0000 7f68d0b16640 -1 rbd::mirror::NamespaceReplayer: 0x562b540b6000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.005+0000 7f68cab0a640 -1 rbd::mirror::MirrorStatusUpdater 0x562b535a30e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.006 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.005+0000 7f68d1317640 -1 rbd::mirror::NamespaceReplayer: 0x562b540b6000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.017 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Stopped 2024-09-18T01:35:05.017 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-18T01:35:05.028 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.0:waiting for process to exit 2024-09-18T01:35:05.028 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:05.059 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.058+0000 7f9fc6137640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 69783) UID: 1000 2024-09-18T01:35:05.059 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.058+0000 7f9fc5135640 -1 rbd::mirror::LeaderWatcher: 0x560533ca5200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:05.059 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fc5135640 -1 rbd::mirror::InstanceWatcher: 0x560533a8e380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6330 object: (2) No such file or directory 2024-09-18T01:35:05.060 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fbe928640 -1 rbd::mirror::InstanceWatcher: 0x560533a8ea80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.060 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fc5936640 -1 rbd::mirror::InstanceWatcher: 0x560533a8e700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6333 object: (2) No such file or directory 2024-09-18T01:35:05.060 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fbf129640 -1 rbd::mirror::InstanceWatcher: 0x560533a8e380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.061 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fb190e640 -1 rbd::mirror::InstanceWatcher: 0x560533a8e700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.061 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fb891c640 -1 rbd::mirror::MirrorStatusUpdater 0x560533ae0480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.061 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fb891c640 -1 rbd::mirror::MirrorStatusUpdater 0x560533ae0360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.061 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fc5135640 -1 rbd::mirror::NamespaceReplayer: 0x560533ade1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.061 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fc5135640 -1 rbd::mirror::NamespaceReplayer: 0x560533ade000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.061 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fb110d640 -1 rbd::mirror::MirrorStatusUpdater 0x560533cc5440 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.061 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fbe928640 -1 rbd::mirror::MirrorStatusUpdater 0x560533ae0120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.061 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fc5936640 -1 rbd::mirror::NamespaceReplayer: 0x560533f5a000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.061 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.059+0000 7f9fc5936640 -1 rbd::mirror::NamespaceReplayer: 0x560533ade1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.062 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.061+0000 7f9fbf129640 -1 rbd::mirror::InstanceWatcher: 0x560533a8e000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.062 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.062+0000 7f9fb811b640 -1 rbd::mirror::MirrorStatusUpdater 0x5605338406c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.062 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.062+0000 7f9fc5135640 -1 rbd::mirror::NamespaceReplayer: 0x5605336f61a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.074 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Stopped 2024-09-18T01:35:05.074 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-18T01:35:05.085 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.3:waiting for process to exit 2024-09-18T01:35:05.085 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:05.115 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.115+0000 7f364e6a9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 42185) UID: 1000 2024-09-18T01:35:05.117 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.115+0000 7f364d6a7640 -1 rbd::mirror::LeaderWatcher: 0x557ef4875200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:05.117 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.116+0000 7f364d6a7640 -1 rbd::mirror::InstanceWatcher: 0x557ef4678380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4547 object: (2) No such file or directory 2024-09-18T01:35:05.117 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.116+0000 7f364d6a7640 -1 rbd::mirror::InstanceWatcher: 0x557ef4b4c000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4547 object: (2) No such file or directory 2024-09-18T01:35:05.117 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.116+0000 7f364d6a7640 -1 rbd::mirror::InstanceWatcher: 0x557ef4678700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4550 object: (2) No such file or directory 2024-09-18T01:35:05.117 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.116+0000 7f364769b640 -1 rbd::mirror::InstanceWatcher: 0x557ef4678380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.117 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.116+0000 7f3646e9a640 -1 rbd::mirror::InstanceWatcher: 0x557ef4b4c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.118 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.116+0000 7f364068d640 -1 rbd::mirror::MirrorStatusUpdater 0x557ef46ec360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.118 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.116+0000 7f364068d640 -1 rbd::mirror::MirrorStatusUpdater 0x557ef46ec5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.118 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.116+0000 7f364dea8640 -1 rbd::mirror::NamespaceReplayer: 0x557ef46ea000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.116+0000 7f364dea8640 -1 rbd::mirror::NamespaceReplayer: 0x557ef46ea1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.116+0000 7f363967f640 -1 rbd::mirror::InstanceWatcher: 0x557ef4678700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.117+0000 7f364769b640 -1 rbd::mirror::MirrorStatusUpdater 0x557ef46ec000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.117+0000 7f364769b640 -1 rbd::mirror::MirrorStatusUpdater 0x557ef46ec120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.117+0000 7f364dea8640 -1 rbd::mirror::NamespaceReplayer: 0x557ef46ea000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.117+0000 7f364d6a7640 -1 rbd::mirror::NamespaceReplayer: 0x557ef46ea1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.117+0000 7f3632e72640 -1 rbd::mirror::MirrorStatusUpdater 0x557ef46ec6c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.117+0000 7f364dea8640 -1 rbd::mirror::NamespaceReplayer: 0x557ef46ea4e0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.117+0000 7f3639e80640 -1 rbd::mirror::MirrorStatusUpdater 0x557ef48a7320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.119 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.117+0000 7f364d6a7640 -1 rbd::mirror::NamespaceReplayer: 0x557ef46ea4e0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.120 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.120+0000 7f364d6a7640 -1 rbd::mirror::LeaderWatcher: 0x557ef45d8c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:05.120 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.120+0000 7f364d6a7640 -1 rbd::mirror::InstanceWatcher: 0x557ef4678000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4547 object: (2) No such file or directory 2024-09-18T01:35:05.121 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.120+0000 7f3646e9a640 -1 rbd::mirror::InstanceWatcher: 0x557ef4678000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.121 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.120+0000 7f3640e8e640 -1 rbd::mirror::MirrorStatusUpdater 0x557ef441c6c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.121 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.120+0000 7f364dea8640 -1 rbd::mirror::NamespaceReplayer: 0x557ef3492680 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.121 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.121+0000 7f364769b640 -1 rbd::mirror::MirrorStatusUpdater 0x557ef43b50e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.121 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi150.stderr:2024-09-18T01:35:05.121+0000 7f364d6a7640 -1 rbd::mirror::NamespaceReplayer: 0x557ef3492680 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.134 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Stopped 2024-09-18T01:35:05.134 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-18T01:35:05.145 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.2:waiting for process to exit 2024-09-18T01:35:05.145 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:05.175 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.175+0000 7f401e55e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 70920) UID: 1000 2024-09-18T01:35:05.177 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.176+0000 7f401dd5d640 -1 rbd::mirror::InstanceWatcher: 0x561cd909c380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6084 object: (2) No such file or directory 2024-09-18T01:35:05.177 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.176+0000 7f401dd5d640 -1 rbd::mirror::InstanceWatcher: 0x561cd909c700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6084 object: (2) No such file or directory 2024-09-18T01:35:05.177 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.176+0000 7f4009d35640 -1 rbd::mirror::InstanceWatcher: 0x561cd955c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.177 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.176+0000 7f4017550640 -1 rbd::mirror::InstanceWatcher: 0x561cd909c380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.176+0000 7f4016d4f640 -1 rbd::mirror::InstanceWatcher: 0x561cd909c700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.176+0000 7f4003528640 -1 rbd::mirror::MirrorStatusUpdater 0x561cd92e4a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.176+0000 7f401dd5d640 -1 rbd::mirror::NamespaceReplayer: 0x561cd9556000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.176+0000 7f4010d43640 -1 rbd::mirror::MirrorStatusUpdater 0x561cd910e360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.176+0000 7f4010d43640 -1 rbd::mirror::MirrorStatusUpdater 0x561cd910e480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.176+0000 7f401dd5d640 -1 rbd::mirror::NamespaceReplayer: 0x561cd910c000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.176+0000 7f401dd5d640 -1 rbd::mirror::NamespaceReplayer: 0x561cd910c1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.177+0000 7f4016d4f640 -1 rbd::mirror::MirrorStatusUpdater 0x561cd910e000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.177+0000 7f4016d4f640 -1 rbd::mirror::MirrorStatusUpdater 0x561cd910e120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.177+0000 7f401d55c640 -1 rbd::mirror::NamespaceReplayer: 0x561cd910c000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.178 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.177+0000 7f401d55c640 -1 rbd::mirror::NamespaceReplayer: 0x561cd910c1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.179 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.178+0000 7f401d55c640 -1 rbd::mirror::LeaderWatcher: 0x561cd9002c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:05.179 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.179+0000 7f401d55c640 -1 rbd::mirror::InstanceWatcher: 0x561cd909c000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6084 object: (2) No such file or directory 2024-09-18T01:35:05.180 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.179+0000 7f4017550640 -1 rbd::mirror::InstanceWatcher: 0x561cd909c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.180 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.179+0000 7f4010542640 -1 rbd::mirror::MirrorStatusUpdater 0x561cd8e566c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.180 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.179+0000 7f401d55c640 -1 rbd::mirror::NamespaceReplayer: 0x561cd7ebe1a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.180 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.179+0000 7f4016d4f640 -1 rbd::mirror::MirrorStatusUpdater 0x561cd8de70e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.180 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi150.stderr:2024-09-18T01:35:05.179+0000 7f401dd5d640 -1 rbd::mirror::NamespaceReplayer: 0x561cd7ebe1a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.192 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Stopped 2024-09-18T01:35:05.192 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-18T01:35:05.202 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.1:waiting for process to exit 2024-09-18T01:35:05.203 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:05.234 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.233+0000 7f1720108640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 57994) UID: 1000 2024-09-18T01:35:05.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.234+0000 7f171f106640 -1 rbd::mirror::LeaderWatcher: 0x55e72c377200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:05.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.234+0000 7f171f106640 -1 rbd::mirror::InstanceWatcher: 0x55e72c170380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5332 object: (2) No such file or directory 2024-09-18T01:35:05.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.234+0000 7f171f106640 -1 rbd::mirror::InstanceWatcher: 0x55e72c170700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5332 object: (2) No such file or directory 2024-09-18T01:35:05.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.234+0000 7f171f106640 -1 rbd::mirror::InstanceWatcher: 0x55e72c170a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5335 object: (2) No such file or directory 2024-09-18T01:35:05.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f17190fa640 -1 rbd::mirror::InstanceWatcher: 0x55e72c170380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.235 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f17190fa640 -1 rbd::mirror::InstanceWatcher: 0x55e72c170700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.236 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f170b0de640 -1 rbd::mirror::InstanceWatcher: 0x55e72c170a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.236 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f17128ed640 -1 rbd::mirror::MirrorStatusUpdater 0x55e72c1b6360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f17128ed640 -1 rbd::mirror::MirrorStatusUpdater 0x55e72c1b6480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f171f106640 -1 rbd::mirror::NamespaceReplayer: 0x55e72c1b4000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f171f106640 -1 rbd::mirror::NamespaceReplayer: 0x55e72c1b41a0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f17188f9640 -1 rbd::mirror::MirrorStatusUpdater 0x55e72c1b6000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f17188f9640 -1 rbd::mirror::MirrorStatusUpdater 0x55e72c1b6120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f171f907640 -1 rbd::mirror::NamespaceReplayer: 0x55e72c1b4000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f171f106640 -1 rbd::mirror::NamespaceReplayer: 0x55e72c1b41a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f17048d1640 -1 rbd::mirror::MirrorStatusUpdater 0x55e72c1b66c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f171f907640 -1 rbd::mirror::NamespaceReplayer: 0x55e72afc4680 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f170b8df640 -1 rbd::mirror::MirrorStatusUpdater 0x55e72c3ad680 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.237 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.235+0000 7f171f106640 -1 rbd::mirror::NamespaceReplayer: 0x55e72afc4680 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.238 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.237+0000 7f171f907640 -1 rbd::mirror::LeaderWatcher: 0x55e72c0c2c00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:05.238 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.238+0000 7f171f907640 -1 rbd::mirror::InstanceWatcher: 0x55e72c170000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5332 object: (2) No such file or directory 2024-09-18T01:35:05.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.238+0000 7f17188f9640 -1 rbd::mirror::InstanceWatcher: 0x55e72c170000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.238+0000 7f17120ec640 -1 rbd::mirror::MirrorStatusUpdater 0x55e72bf266c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.238+0000 7f171f907640 -1 rbd::mirror::NamespaceReplayer: 0x55e72c9e0000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.239+0000 7f17190fa640 -1 rbd::mirror::MirrorStatusUpdater 0x55e72bec7200 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.239 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi150.stderr:2024-09-18T01:35:05.239+0000 7f171f106640 -1 rbd::mirror::NamespaceReplayer: 0x55e72c9e0000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.252 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Stopped 2024-09-18T01:35:05.252 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2024-09-18T01:35:05.262 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.0:waiting for process to exit 2024-09-18T01:35:05.263 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:05.294 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.293+0000 7f4f889e9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 71746) UID: 1000 2024-09-18T01:35:05.294 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.294+0000 7f4f881e8640 -1 rbd::mirror::LeaderWatcher: 0x55b32b795200 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:05.294 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.294+0000 7f4f881e8640 -1 rbd::mirror::InstanceWatcher: 0x55b32b5a2380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6128 object: (2) No such file or directory 2024-09-18T01:35:05.295 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.294+0000 7f4f881e8640 -1 rbd::mirror::InstanceWatcher: 0x55b32ba82000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6128 object: (2) No such file or directory 2024-09-18T01:35:05.295 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.294+0000 7f4f879e7640 -1 rbd::mirror::InstanceWatcher: 0x55b32b5a2700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6131 object: (2) No such file or directory 2024-09-18T01:35:05.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.294+0000 7f4f819db640 -1 rbd::mirror::InstanceWatcher: 0x55b32b5a2380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.294+0000 7f4f811da640 -1 rbd::mirror::InstanceWatcher: 0x55b32ba82000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.295+0000 7f4f739bf640 -1 rbd::mirror::InstanceWatcher: 0x55b32b5a2700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.295+0000 7f4f819db640 -1 rbd::mirror::MirrorStatusUpdater 0x55b32b60e000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.295+0000 7f4f819db640 -1 rbd::mirror::MirrorStatusUpdater 0x55b32b60e120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.295+0000 7f4f879e7640 -1 rbd::mirror::NamespaceReplayer: 0x55b32b5a41a0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.295+0000 7f4f879e7640 -1 rbd::mirror::NamespaceReplayer: 0x55b32b5a4340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.295+0000 7f4f6d9b3640 -1 rbd::mirror::MirrorStatusUpdater 0x55b32b60e480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.296 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.295+0000 7f4f881e8640 -1 rbd::mirror::NamespaceReplayer: 0x55b32a3b8680 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.297 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.295+0000 7f4f741c0640 -1 rbd::mirror::MirrorStatusUpdater 0x55b32b790fc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.297 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.295+0000 7f4f879e7640 -1 rbd::mirror::NamespaceReplayer: 0x55b32a3b8680 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.297 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.297+0000 7f4f879e7640 -1 rbd::mirror::LeaderWatcher: 0x55b32b4fec00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2024-09-18T01:35:05.297 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.297+0000 7f4f879e7640 -1 rbd::mirror::InstanceWatcher: 0x55b32b5a2000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6128 object: (2) No such file or directory 2024-09-18T01:35:05.298 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.298+0000 7f4f811da640 -1 rbd::mirror::InstanceWatcher: 0x55b32b5a2000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2024-09-18T01:35:05.299 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.298+0000 7f4f819db640 -1 rbd::mirror::MirrorStatusUpdater 0x55b32b2e90e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2024-09-18T01:35:05.299 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi150.stderr:2024-09-18T01:35:05.298+0000 7f4f879e7640 -1 rbd::mirror::NamespaceReplayer: 0x55b32be00000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2024-09-18T01:35:05.309 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Stopped 2024-09-18T01:35:05.310 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-18T01:35:05.321 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2024-09-18T01:35:05.321 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T01:35:05.635 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T01:35:05.635 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-09-18T01:35:05.648 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":1575,"stamp":"2024-09-18T01:35:05.633454+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4481232,"kb_used_data":15892,"kb_used_omap":4,"kb_used_meta":4465275,"kb_avail":278634288,"statfs":{"total":289910292480,"available":285321510912,"internally_reserved":0,"allocated":16273408,"data_stored":9092384,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4789,"internal_metadata":4572441931},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536847413,"num_objects":-164,"num_object_clones":0,"num_object_copies":-328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-164,"num_whiteouts":0,"num_read":-114272,"num_read_kb":-39563432,"num_write":-656665,"num_write_kb":-245019078,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":0,"num_keys_recovered":-8,"num_objects_omap":-14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.505533"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.131245+0000","last_change":"2024-09-18T00:57:54.172397+0000","last_active":"2024-09-18T01:34:59.131245+0000","last_peered":"2024-09-18T01:34:59.131245+0000","last_clean":"2024-09-18T01:34:59.131245+0000","last_became_active":"2024-09-18T00:57:52.161899+0000","last_became_peered":"2024-09-18T00:57:52.161899+0000","last_unstale":"2024-09-18T01:34:59.131245+0000","last_undegraded":"2024-09-18T01:34:59.131245+0000","last_fullsized":"2024-09-18T01:34:59.131245+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:00:53.878037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075298400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.132482+0000","last_change":"2024-09-18T00:57:54.172095+0000","last_active":"2024-09-18T01:34:59.132482+0000","last_peered":"2024-09-18T01:34:59.132482+0000","last_clean":"2024-09-18T01:34:59.132482+0000","last_became_active":"2024-09-18T00:57:52.159679+0000","last_became_peered":"2024-09-18T00:57:52.159679+0000","last_unstale":"2024-09-18T01:34:59.132482+0000","last_undegraded":"2024-09-18T01:34:59.132482+0000","last_fullsized":"2024-09-18T01:34:59.132482+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:43:58.913774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054889900000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.132545+0000","last_change":"2024-09-18T00:57:54.172691+0000","last_active":"2024-09-18T01:34:59.132545+0000","last_peered":"2024-09-18T01:34:59.132545+0000","last_clean":"2024-09-18T01:34:59.132545+0000","last_became_active":"2024-09-18T00:57:52.160197+0000","last_became_peered":"2024-09-18T00:57:52.160197+0000","last_unstale":"2024-09-18T01:34:59.132545+0000","last_undegraded":"2024-09-18T01:34:59.132545+0000","last_fullsized":"2024-09-18T01:34:59.132545+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:26:56.936297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096762200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.130846+0000","last_change":"2024-09-18T00:57:54.172438+0000","last_active":"2024-09-18T01:34:59.130846+0000","last_peered":"2024-09-18T01:34:59.130846+0000","last_clean":"2024-09-18T01:34:59.130846+0000","last_became_active":"2024-09-18T00:57:52.159800+0000","last_became_peered":"2024-09-18T00:57:52.159800+0000","last_unstale":"2024-09-18T01:34:59.130846+0000","last_undegraded":"2024-09-18T01:34:59.130846+0000","last_fullsized":"2024-09-18T01:34:59.130846+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:09:57.052840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00074951200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":916,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.370415+0000","last_change":"2024-09-18T00:57:54.170233+0000","last_active":"2024-09-18T01:34:59.370415+0000","last_peered":"2024-09-18T01:34:59.370415+0000","last_clean":"2024-09-18T01:34:59.370415+0000","last_became_active":"2024-09-18T00:57:52.159283+0000","last_became_peered":"2024-09-18T00:57:52.159283+0000","last_unstale":"2024-09-18T01:34:59.370415+0000","last_undegraded":"2024-09-18T01:34:59.370415+0000","last_fullsized":"2024-09-18T01:34:59.370415+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:56:30.005098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00113901,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":910,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.138972+0000","last_change":"2024-09-18T00:57:54.167662+0000","last_active":"2024-09-18T01:34:59.138972+0000","last_peered":"2024-09-18T01:34:59.138972+0000","last_clean":"2024-09-18T01:34:59.138972+0000","last_became_active":"2024-09-18T00:57:52.160105+0000","last_became_peered":"2024-09-18T00:57:52.160105+0000","last_unstale":"2024-09-18T01:34:59.138972+0000","last_undegraded":"2024-09-18T01:34:59.138972+0000","last_fullsized":"2024-09-18T01:34:59.138972+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:10:35.692324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041490800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.139784+0000","last_change":"2024-09-18T00:57:54.167698+0000","last_active":"2024-09-18T01:34:59.139784+0000","last_peered":"2024-09-18T01:34:59.139784+0000","last_clean":"2024-09-18T01:34:59.139784+0000","last_became_active":"2024-09-18T00:57:52.159792+0000","last_became_peered":"2024-09-18T00:57:52.159792+0000","last_unstale":"2024-09-18T01:34:59.139784+0000","last_undegraded":"2024-09-18T01:34:59.139784+0000","last_fullsized":"2024-09-18T01:34:59.139784+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:42:59.227183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045536600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":912,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.131787+0000","last_change":"2024-09-18T00:57:54.172073+0000","last_active":"2024-09-18T01:34:59.131787+0000","last_peered":"2024-09-18T01:34:59.131787+0000","last_clean":"2024-09-18T01:34:59.131787+0000","last_became_active":"2024-09-18T00:57:52.158758+0000","last_became_peered":"2024-09-18T00:57:52.158758+0000","last_unstale":"2024-09-18T01:34:59.131787+0000","last_undegraded":"2024-09-18T01:34:59.131787+0000","last_fullsized":"2024-09-18T01:34:59.131787+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:44:25.148787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053750599999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'97","reported_seq":1040,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.131365+0000","last_change":"2024-09-18T00:57:47.137803+0000","last_active":"2024-09-18T01:34:59.131365+0000","last_peered":"2024-09-18T01:34:59.131365+0000","last_clean":"2024-09-18T01:34:59.131365+0000","last_became_active":"2024-09-18T00:57:47.136874+0000","last_became_peered":"2024-09-18T00:57:47.136874+0000","last_unstale":"2024-09-18T01:34:59.131365+0000","last_undegraded":"2024-09-18T01:34:59.131365+0000","last_fullsized":"2024-09-18T01:34:59.131365+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_clean_scrub_stamp":"2024-09-18T00:57:46.126096+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T04:09:27.863635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706115,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1427104,"kb_used_data":5260,"kb_used_omap":1,"kb_used_meta":1421822,"kb_avail":92944736,"statfs":{"total":96636764160,"available":95175409664,"internally_reserved":0,"allocated":5386240,"data_stored":2637216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1455946181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738818,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1377560,"kb_used_data":5316,"kb_used_omap":1,"kb_used_meta":1372222,"kb_avail":92994280,"statfs":{"total":96636764160,"available":95226142720,"internally_reserved":0,"allocated":5443584,"data_stored":3227584,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":1405155781},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738820,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1676568,"kb_used_data":5316,"kb_used_omap":1,"kb_used_meta":1671230,"kb_avail":92695272,"statfs":{"total":96636764160,"available":94919958528,"internally_reserved":0,"allocated":5443584,"data_stored":3227584,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1599,"internal_metadata":1711339969},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:35:05.649 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T01:35:05.928 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T01:35:05.929 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-09-18T01:35:05.941 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":1575,"stamp":"2024-09-18T01:35:05.633454+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4481232,"kb_used_data":15892,"kb_used_omap":4,"kb_used_meta":4465275,"kb_avail":278634288,"statfs":{"total":289910292480,"available":285321510912,"internally_reserved":0,"allocated":16273408,"data_stored":9092384,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4789,"internal_metadata":4572441931},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536847413,"num_objects":-164,"num_object_clones":0,"num_object_copies":-328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-164,"num_whiteouts":0,"num_read":-114272,"num_read_kb":-39563432,"num_write":-656665,"num_write_kb":-245019078,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":0,"num_keys_recovered":-8,"num_objects_omap":-14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.505533"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.131245+0000","last_change":"2024-09-18T00:57:54.172397+0000","last_active":"2024-09-18T01:34:59.131245+0000","last_peered":"2024-09-18T01:34:59.131245+0000","last_clean":"2024-09-18T01:34:59.131245+0000","last_became_active":"2024-09-18T00:57:52.161899+0000","last_became_peered":"2024-09-18T00:57:52.161899+0000","last_unstale":"2024-09-18T01:34:59.131245+0000","last_undegraded":"2024-09-18T01:34:59.131245+0000","last_fullsized":"2024-09-18T01:34:59.131245+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:00:53.878037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075298400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.132482+0000","last_change":"2024-09-18T00:57:54.172095+0000","last_active":"2024-09-18T01:34:59.132482+0000","last_peered":"2024-09-18T01:34:59.132482+0000","last_clean":"2024-09-18T01:34:59.132482+0000","last_became_active":"2024-09-18T00:57:52.159679+0000","last_became_peered":"2024-09-18T00:57:52.159679+0000","last_unstale":"2024-09-18T01:34:59.132482+0000","last_undegraded":"2024-09-18T01:34:59.132482+0000","last_fullsized":"2024-09-18T01:34:59.132482+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:43:58.913774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054889900000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.132545+0000","last_change":"2024-09-18T00:57:54.172691+0000","last_active":"2024-09-18T01:34:59.132545+0000","last_peered":"2024-09-18T01:34:59.132545+0000","last_clean":"2024-09-18T01:34:59.132545+0000","last_became_active":"2024-09-18T00:57:52.160197+0000","last_became_peered":"2024-09-18T00:57:52.160197+0000","last_unstale":"2024-09-18T01:34:59.132545+0000","last_undegraded":"2024-09-18T01:34:59.132545+0000","last_fullsized":"2024-09-18T01:34:59.132545+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:26:56.936297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096762200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.130846+0000","last_change":"2024-09-18T00:57:54.172438+0000","last_active":"2024-09-18T01:34:59.130846+0000","last_peered":"2024-09-18T01:34:59.130846+0000","last_clean":"2024-09-18T01:34:59.130846+0000","last_became_active":"2024-09-18T00:57:52.159800+0000","last_became_peered":"2024-09-18T00:57:52.159800+0000","last_unstale":"2024-09-18T01:34:59.130846+0000","last_undegraded":"2024-09-18T01:34:59.130846+0000","last_fullsized":"2024-09-18T01:34:59.130846+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:09:57.052840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00074951200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":916,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.370415+0000","last_change":"2024-09-18T00:57:54.170233+0000","last_active":"2024-09-18T01:34:59.370415+0000","last_peered":"2024-09-18T01:34:59.370415+0000","last_clean":"2024-09-18T01:34:59.370415+0000","last_became_active":"2024-09-18T00:57:52.159283+0000","last_became_peered":"2024-09-18T00:57:52.159283+0000","last_unstale":"2024-09-18T01:34:59.370415+0000","last_undegraded":"2024-09-18T01:34:59.370415+0000","last_fullsized":"2024-09-18T01:34:59.370415+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:56:30.005098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00113901,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":910,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.138972+0000","last_change":"2024-09-18T00:57:54.167662+0000","last_active":"2024-09-18T01:34:59.138972+0000","last_peered":"2024-09-18T01:34:59.138972+0000","last_clean":"2024-09-18T01:34:59.138972+0000","last_became_active":"2024-09-18T00:57:52.160105+0000","last_became_peered":"2024-09-18T00:57:52.160105+0000","last_unstale":"2024-09-18T01:34:59.138972+0000","last_undegraded":"2024-09-18T01:34:59.138972+0000","last_fullsized":"2024-09-18T01:34:59.138972+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:10:35.692324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041490800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.139784+0000","last_change":"2024-09-18T00:57:54.167698+0000","last_active":"2024-09-18T01:34:59.139784+0000","last_peered":"2024-09-18T01:34:59.139784+0000","last_clean":"2024-09-18T01:34:59.139784+0000","last_became_active":"2024-09-18T00:57:52.159792+0000","last_became_peered":"2024-09-18T00:57:52.159792+0000","last_unstale":"2024-09-18T01:34:59.139784+0000","last_undegraded":"2024-09-18T01:34:59.139784+0000","last_fullsized":"2024-09-18T01:34:59.139784+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:42:59.227183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045536600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":912,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.131787+0000","last_change":"2024-09-18T00:57:54.172073+0000","last_active":"2024-09-18T01:34:59.131787+0000","last_peered":"2024-09-18T01:34:59.131787+0000","last_clean":"2024-09-18T01:34:59.131787+0000","last_became_active":"2024-09-18T00:57:52.158758+0000","last_became_peered":"2024-09-18T00:57:52.158758+0000","last_unstale":"2024-09-18T01:34:59.131787+0000","last_undegraded":"2024-09-18T01:34:59.131787+0000","last_fullsized":"2024-09-18T01:34:59.131787+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:44:25.148787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053750599999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'97","reported_seq":1040,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.131365+0000","last_change":"2024-09-18T00:57:47.137803+0000","last_active":"2024-09-18T01:34:59.131365+0000","last_peered":"2024-09-18T01:34:59.131365+0000","last_clean":"2024-09-18T01:34:59.131365+0000","last_became_active":"2024-09-18T00:57:47.136874+0000","last_became_peered":"2024-09-18T00:57:47.136874+0000","last_unstale":"2024-09-18T01:34:59.131365+0000","last_undegraded":"2024-09-18T01:34:59.131365+0000","last_fullsized":"2024-09-18T01:34:59.131365+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_clean_scrub_stamp":"2024-09-18T00:57:46.126096+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T04:09:27.863635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706115,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1427104,"kb_used_data":5260,"kb_used_omap":1,"kb_used_meta":1421822,"kb_avail":92944736,"statfs":{"total":96636764160,"available":95175409664,"internally_reserved":0,"allocated":5386240,"data_stored":2637216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1455946181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738818,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1377560,"kb_used_data":5316,"kb_used_omap":1,"kb_used_meta":1372222,"kb_avail":92994280,"statfs":{"total":96636764160,"available":95226142720,"internally_reserved":0,"allocated":5443584,"data_stored":3227584,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":1405155781},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738820,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1676568,"kb_used_data":5316,"kb_used_omap":1,"kb_used_meta":1671230,"kb_avail":92695272,"statfs":{"total":96636764160,"available":94919958528,"internally_reserved":0,"allocated":5443584,"data_stored":3227584,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1599,"internal_metadata":1711339969},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:35:05.942 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2024-09-18T01:35:05.942 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T01:35:06.217 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T01:35:06.217 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-09-18T01:35:06.229 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":1575,"stamp":"2024-09-18T01:35:05.633454+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4481232,"kb_used_data":15892,"kb_used_omap":4,"kb_used_meta":4465275,"kb_avail":278634288,"statfs":{"total":289910292480,"available":285321510912,"internally_reserved":0,"allocated":16273408,"data_stored":9092384,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4789,"internal_metadata":4572441931},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536847413,"num_objects":-164,"num_object_clones":0,"num_object_copies":-328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-164,"num_whiteouts":0,"num_read":-114272,"num_read_kb":-39563432,"num_write":-656665,"num_write_kb":-245019078,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":0,"num_keys_recovered":-8,"num_objects_omap":-14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.505533"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.131245+0000","last_change":"2024-09-18T00:57:54.172397+0000","last_active":"2024-09-18T01:34:59.131245+0000","last_peered":"2024-09-18T01:34:59.131245+0000","last_clean":"2024-09-18T01:34:59.131245+0000","last_became_active":"2024-09-18T00:57:52.161899+0000","last_became_peered":"2024-09-18T00:57:52.161899+0000","last_unstale":"2024-09-18T01:34:59.131245+0000","last_undegraded":"2024-09-18T01:34:59.131245+0000","last_fullsized":"2024-09-18T01:34:59.131245+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:00:53.878037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075298400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.132482+0000","last_change":"2024-09-18T00:57:54.172095+0000","last_active":"2024-09-18T01:34:59.132482+0000","last_peered":"2024-09-18T01:34:59.132482+0000","last_clean":"2024-09-18T01:34:59.132482+0000","last_became_active":"2024-09-18T00:57:52.159679+0000","last_became_peered":"2024-09-18T00:57:52.159679+0000","last_unstale":"2024-09-18T01:34:59.132482+0000","last_undegraded":"2024-09-18T01:34:59.132482+0000","last_fullsized":"2024-09-18T01:34:59.132482+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:43:58.913774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054889900000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.132545+0000","last_change":"2024-09-18T00:57:54.172691+0000","last_active":"2024-09-18T01:34:59.132545+0000","last_peered":"2024-09-18T01:34:59.132545+0000","last_clean":"2024-09-18T01:34:59.132545+0000","last_became_active":"2024-09-18T00:57:52.160197+0000","last_became_peered":"2024-09-18T00:57:52.160197+0000","last_unstale":"2024-09-18T01:34:59.132545+0000","last_undegraded":"2024-09-18T01:34:59.132545+0000","last_fullsized":"2024-09-18T01:34:59.132545+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:26:56.936297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096762200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.130846+0000","last_change":"2024-09-18T00:57:54.172438+0000","last_active":"2024-09-18T01:34:59.130846+0000","last_peered":"2024-09-18T01:34:59.130846+0000","last_clean":"2024-09-18T01:34:59.130846+0000","last_became_active":"2024-09-18T00:57:52.159800+0000","last_became_peered":"2024-09-18T00:57:52.159800+0000","last_unstale":"2024-09-18T01:34:59.130846+0000","last_undegraded":"2024-09-18T01:34:59.130846+0000","last_fullsized":"2024-09-18T01:34:59.130846+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:09:57.052840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00074951200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":916,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.370415+0000","last_change":"2024-09-18T00:57:54.170233+0000","last_active":"2024-09-18T01:34:59.370415+0000","last_peered":"2024-09-18T01:34:59.370415+0000","last_clean":"2024-09-18T01:34:59.370415+0000","last_became_active":"2024-09-18T00:57:52.159283+0000","last_became_peered":"2024-09-18T00:57:52.159283+0000","last_unstale":"2024-09-18T01:34:59.370415+0000","last_undegraded":"2024-09-18T01:34:59.370415+0000","last_fullsized":"2024-09-18T01:34:59.370415+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:56:30.005098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00113901,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":910,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.138972+0000","last_change":"2024-09-18T00:57:54.167662+0000","last_active":"2024-09-18T01:34:59.138972+0000","last_peered":"2024-09-18T01:34:59.138972+0000","last_clean":"2024-09-18T01:34:59.138972+0000","last_became_active":"2024-09-18T00:57:52.160105+0000","last_became_peered":"2024-09-18T00:57:52.160105+0000","last_unstale":"2024-09-18T01:34:59.138972+0000","last_undegraded":"2024-09-18T01:34:59.138972+0000","last_fullsized":"2024-09-18T01:34:59.138972+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:10:35.692324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041490800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":911,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.139784+0000","last_change":"2024-09-18T00:57:54.167698+0000","last_active":"2024-09-18T01:34:59.139784+0000","last_peered":"2024-09-18T01:34:59.139784+0000","last_clean":"2024-09-18T01:34:59.139784+0000","last_became_active":"2024-09-18T00:57:52.159792+0000","last_became_peered":"2024-09-18T00:57:52.159792+0000","last_unstale":"2024-09-18T01:34:59.139784+0000","last_undegraded":"2024-09-18T01:34:59.139784+0000","last_fullsized":"2024-09-18T01:34:59.139784+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:42:59.227183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045536600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":912,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.131787+0000","last_change":"2024-09-18T00:57:54.172073+0000","last_active":"2024-09-18T01:34:59.131787+0000","last_peered":"2024-09-18T01:34:59.131787+0000","last_clean":"2024-09-18T01:34:59.131787+0000","last_became_active":"2024-09-18T00:57:52.158758+0000","last_became_peered":"2024-09-18T00:57:52.158758+0000","last_unstale":"2024-09-18T01:34:59.131787+0000","last_undegraded":"2024-09-18T01:34:59.131787+0000","last_fullsized":"2024-09-18T01:34:59.131787+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:44:25.148787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053750599999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'97","reported_seq":1040,"reported_epoch":455,"state":"active+clean","last_fresh":"2024-09-18T01:34:59.131365+0000","last_change":"2024-09-18T00:57:47.137803+0000","last_active":"2024-09-18T01:34:59.131365+0000","last_peered":"2024-09-18T01:34:59.131365+0000","last_clean":"2024-09-18T01:34:59.131365+0000","last_became_active":"2024-09-18T00:57:47.136874+0000","last_became_peered":"2024-09-18T00:57:47.136874+0000","last_unstale":"2024-09-18T01:34:59.131365+0000","last_undegraded":"2024-09-18T01:34:59.131365+0000","last_fullsized":"2024-09-18T01:34:59.131365+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_clean_scrub_stamp":"2024-09-18T00:57:46.126096+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T04:09:27.863635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706115,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1427104,"kb_used_data":5260,"kb_used_omap":1,"kb_used_meta":1421822,"kb_avail":92944736,"statfs":{"total":96636764160,"available":95175409664,"internally_reserved":0,"allocated":5386240,"data_stored":2637216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1455946181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738818,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1377560,"kb_used_data":5316,"kb_used_omap":1,"kb_used_meta":1372222,"kb_avail":92994280,"statfs":{"total":96636764160,"available":95226142720,"internally_reserved":0,"allocated":5443584,"data_stored":3227584,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":1405155781},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738820,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1676568,"kb_used_data":5316,"kb_used_omap":1,"kb_used_meta":1671230,"kb_avail":92695272,"statfs":{"total":96636764160,"available":94919958528,"internally_reserved":0,"allocated":5443584,"data_stored":3227584,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1599,"internal_metadata":1711339969},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:35:06.230 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-09-18T01:35:06.504 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T01:35:06.505 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":457,"fsid":"134b3e88-b82a-4f51-8a00-69fd22a05e09","created":"2024-09-18T00:57:35.765780+0000","modified":"2024-09-18T01:35:04.140203+0000","last_up_change":"2024-09-18T00:57:45.120993+0000","last_in_change":"2024-09-18T00:57:36.604173+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-18T00:57:45.280415+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-18T00:57:50.851259+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"dda8f346-bbdb-42fd-876f-c6cdec5d1826","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6801","nonce":204393861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6803","nonce":204393861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6807","nonce":204393861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":204393861},{"type":"v1","addr":"172.21.15.150:6805","nonce":204393861}]},"public_addr":"172.21.15.150:6801/204393861","cluster_addr":"172.21.15.150:6803/204393861","heartbeat_back_addr":"172.21.15.150:6807/204393861","heartbeat_front_addr":"172.21.15.150:6805/204393861","state":["exists","up"]},{"osd":1,"uuid":"32875857-ffe6-4037-be43-509984a1afdf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6817","nonce":1827386980}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6819","nonce":1827386980}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6823","nonce":1827386980}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":1827386980},{"type":"v1","addr":"172.21.15.150:6821","nonce":1827386980}]},"public_addr":"172.21.15.150:6817/1827386980","cluster_addr":"172.21.15.150:6819/1827386980","heartbeat_back_addr":"172.21.15.150:6823/1827386980","heartbeat_front_addr":"172.21.15.150:6821/1827386980","state":["exists","up"]},{"osd":2,"uuid":"0095f5ec-4942-4407-a217-6923831ae2ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":29,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6809","nonce":3623182795}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6811","nonce":3623182795}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6815","nonce":3623182795}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3623182795},{"type":"v1","addr":"172.21.15.150:6813","nonce":3623182795}]},"public_addr":"172.21.15.150:6809/3623182795","cluster_addr":"172.21.15.150:6811/3623182795","heartbeat_back_addr":"172.21.15.150:6815/3623182795","heartbeat_front_addr":"172.21.15.150:6813/3623182795","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:42.398707+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:42.782487+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:43.343405+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.150:0/3664315924":"2024-09-18T02:34:58.813621+0000","172.21.15.150:0/2350365732":"2024-09-18T02:34:15.722470+0000","172.21.15.150:0/4260850020":"2024-09-18T02:33:40.161844+0000","172.21.15.150:0/7308716":"2024-09-18T02:26:20.921994+0000","172.21.15.150:0/545696895":"2024-09-18T02:21:31.922209+0000","172.21.15.150:0/336108186":"2024-09-18T02:21:24.906101+0000","172.21.15.150:0/1003317196":"2024-09-18T02:21:03.839015+0000","172.21.15.150:0/3444865883":"2024-09-18T02:20:49.778913+0000","172.21.15.150:0/2293037173":"2024-09-18T02:20:35.731833+0000","172.21.15.150:0/3077539209":"2024-09-18T02:22:21.714333+0000","172.21.15.150:0/1343297847":"2024-09-18T02:20:28.975715+0000","172.21.15.150:0/1828206662":"2024-09-18T02:23:41.743998+0000","172.21.15.150:0/3705290304":"2024-09-18T02:20:21.726081+0000","172.21.15.150:0/3306313275":"2024-09-18T02:31:39.217036+0000","172.21.15.150:0/2181325537":"2024-09-18T02:25:09.905803+0000","172.21.15.150:0/2148706791":"2024-09-18T02:20:14.694278+0000","172.21.15.150:0/963210116":"2024-09-18T02:07:16.782264+0000","172.21.15.150:0/3200532796":"2024-09-18T02:20:56.852308+0000","172.21.15.150:0/3345786627":"2024-09-18T02:20:07.677639+0000","172.21.15.150:0/3684348806":"2024-09-18T02:01:12.169169+0000","172.21.15.150:0/398873641":"2024-09-18T02:02:00.183683+0000","172.21.15.150:0/904508081":"2024-09-18T02:20:42.787941+0000","172.21.15.150:0/278915892":"2024-09-18T02:20:01.089038+0000","172.21.15.150:0/3099300538":"2024-09-18T02:32:16.142509+0000","172.21.15.150:0/3072665015":"2024-09-18T02:02:55.424983+0000","172.21.15.150:0/601270985":"2024-09-18T02:22:45.787106+0000","172.21.15.150:0/143582226":"2024-09-18T02:29:34.737250+0000","172.21.15.150:0/2005224777":"2024-09-18T02:21:40.590210+0000","172.21.15.150:0/3539427161":"2024-09-18T02:23:01.838894+0000","172.21.15.150:0/2134097732":"2024-09-18T02:21:17.909412+0000","172.21.15.150:0/3878274367":"2024-09-18T02:23:26.906382+0000","172.21.15.150:0/3083160202":"2024-09-18T02:24:20.794369+0000","172.21.15.150:0/2357234897":"2024-09-18T02:00:25.284292+0000","172.21.15.150:0/3904058792":"2024-09-18T02:03:43.273720+0000","172.21.15.150:0/1108670918":"2024-09-18T02:04:33.579911+0000","172.21.15.150:0/1963601972":"2024-09-18T02:21:10.887030+0000","172.21.15.150:0/3363220221":"2024-09-18T02:25:52.838880+0000","172.21.15.150:0/3750759807":"2024-09-18T02:06:20.244868+0000","172.21.15.150:0/570257522":"2024-09-18T02:23:48.757224+0000","172.21.15.150:0/2634180191":"2024-09-18T02:26:06.884324+0000","172.21.15.150:0/3972729722":"2024-09-18T02:05:26.654890+0000","172.21.15.150:0/3269402484":"2024-09-18T02:21:56.857184+0000","172.21.15.150:0/2807427580":"2024-09-18T02:23:09.837075+0000","172.21.15.150:0/2771220618":"2024-09-18T02:19:19.982031+0000","172.21.15.150:0/3876822973":"2024-09-18T02:08:09.198207+0000","172.21.15.150:0/1697923374":"2024-09-18T02:22:53.824680+0000","172.21.15.150:0/770981780":"2024-09-18T02:30:17.834433+0000","172.21.15.150:0/2578867928":"2024-09-18T02:22:05.682978+0000","172.21.15.150:0/3302628399":"2024-09-18T02:22:13.702300+0000","172.21.15.150:0/555578267":"2024-09-18T02:22:29.723361+0000","172.21.15.150:0/3089126902":"2024-09-18T02:24:53.106499+0000","172.21.15.150:0/1663415734":"2024-09-18T02:28:56.509128+0000","172.21.15.150:0/2629642377":"2024-09-18T02:25:17.916209+0000","172.21.15.150:0/1613635335":"2024-09-18T02:22:37.758889+0000","172.21.15.150:0/2458392347":"2024-09-18T02:19:52.193239+0000","172.21.15.150:0/2539561660":"2024-09-18T02:26:13.889686+0000","172.21.15.150:0/3372755991":"2024-09-18T02:23:18.090487+0000","172.21.15.150:0/127439583":"2024-09-18T02:25:24.937622+0000","172.21.15.150:0/3849800300":"2024-09-18T02:23:34.928786+0000","172.21.15.150:0/2776295307":"2024-09-18T02:21:48.619515+0000","172.21.15.150:0/240075053":"2024-09-18T02:24:44.849020+0000","172.21.15.150:0/3295749822":"2024-09-18T02:31:02.291757+0000","172.21.15.150:0/2521687753":"2024-09-18T02:24:27.804467+0000","172.21.15.150:0/2538158354":"2024-09-18T02:24:36.836196+0000","172.21.15.150:0/1522568378":"2024-09-18T02:25:31.778674+0000","172.21.15.150:0/826252025":"2024-09-18T02:25:38.809889+0000","172.21.15.150:0/1814952842":"2024-09-18T02:25:01.934268+0000","172.21.15.150:0/4072585792":"2024-09-18T02:25:45.826674+0000","172.21.15.150:0/3747748237":"2024-09-18T02:32:58.153153+0000","172.21.15.150:0/1566591542":"2024-09-18T02:25:59.844887+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T01:35:07.518 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2024-09-18T01:35:07.518 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:35:07.653 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2024-09-18T01:35:07.653 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:35:07.653 INFO:teuthology.orchestra.run.smithi150.stdout:} 2024-09-18T01:35:07.664 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2024-09-18T01:35:07.940 INFO:teuthology.orchestra.run.smithi150.stderr:instructed osd(s) 0 to deep-scrub 2024-09-18T01:35:07.953 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2024-09-18T01:35:07.954 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:35:08.076 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2024-09-18T01:35:08.076 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:35:08.077 INFO:teuthology.orchestra.run.smithi150.stdout:} 2024-09-18T01:35:08.087 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2024-09-18T01:35:08.363 INFO:teuthology.orchestra.run.smithi150.stderr:instructed osd(s) 1 to deep-scrub 2024-09-18T01:35:08.376 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2024-09-18T01:35:08.376 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:35:08.498 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2024-09-18T01:35:08.498 INFO:teuthology.orchestra.run.smithi150.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:35:08.498 INFO:teuthology.orchestra.run.smithi150.stdout:} 2024-09-18T01:35:08.509 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2024-09-18T01:35:08.784 INFO:teuthology.orchestra.run.smithi150.stderr:instructed osd(s) 2 to deep-scrub 2024-09-18T01:35:08.797 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T01:35:09.071 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T01:35:09.072 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-09-18T01:35:09.084 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":1576,"stamp":"2024-09-18T01:35:07.633838+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4499364,"kb_used_data":15972,"kb_used_omap":4,"kb_used_meta":4483323,"kb_avail":278616156,"statfs":{"total":289910292480,"available":285302943744,"internally_reserved":0,"allocated":16355328,"data_stored":9133570,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4789,"internal_metadata":4590923083},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536847413,"num_objects":-164,"num_object_clones":0,"num_object_copies":-328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-164,"num_whiteouts":0,"num_read":-114272,"num_read_kb":-39563432,"num_write":-656665,"num_write_kb":-245019078,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":0,"num_keys_recovered":-8,"num_objects_omap":-14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002003"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":915,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:04.149438+0000","last_change":"2024-09-18T00:57:54.172397+0000","last_active":"2024-09-18T01:35:04.149438+0000","last_peered":"2024-09-18T01:35:04.149438+0000","last_clean":"2024-09-18T01:35:04.149438+0000","last_became_active":"2024-09-18T00:57:52.161899+0000","last_became_peered":"2024-09-18T00:57:52.161899+0000","last_unstale":"2024-09-18T01:35:04.149438+0000","last_undegraded":"2024-09-18T01:35:04.149438+0000","last_fullsized":"2024-09-18T01:35:04.149438+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:00:53.878037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075298400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":915,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:04.149123+0000","last_change":"2024-09-18T00:57:54.172095+0000","last_active":"2024-09-18T01:35:04.149123+0000","last_peered":"2024-09-18T01:35:04.149123+0000","last_clean":"2024-09-18T01:35:04.149123+0000","last_became_active":"2024-09-18T00:57:52.159679+0000","last_became_peered":"2024-09-18T00:57:52.159679+0000","last_unstale":"2024-09-18T01:35:04.149123+0000","last_undegraded":"2024-09-18T01:35:04.149123+0000","last_fullsized":"2024-09-18T01:35:04.149123+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T03:43:58.913774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054889900000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":915,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:04.147987+0000","last_change":"2024-09-18T00:57:54.172691+0000","last_active":"2024-09-18T01:35:04.147987+0000","last_peered":"2024-09-18T01:35:04.147987+0000","last_clean":"2024-09-18T01:35:04.147987+0000","last_became_active":"2024-09-18T00:57:52.160197+0000","last_became_peered":"2024-09-18T00:57:52.160197+0000","last_unstale":"2024-09-18T01:35:04.147987+0000","last_undegraded":"2024-09-18T01:35:04.147987+0000","last_fullsized":"2024-09-18T01:35:04.147987+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:26:56.936297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096762200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":915,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:04.148129+0000","last_change":"2024-09-18T00:57:54.172438+0000","last_active":"2024-09-18T01:35:04.148129+0000","last_peered":"2024-09-18T01:35:04.148129+0000","last_clean":"2024-09-18T01:35:04.148129+0000","last_became_active":"2024-09-18T00:57:52.159800+0000","last_became_peered":"2024-09-18T00:57:52.159800+0000","last_unstale":"2024-09-18T01:35:04.148129+0000","last_undegraded":"2024-09-18T01:35:04.148129+0000","last_fullsized":"2024-09-18T01:35:04.148129+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:09:57.052840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00074951200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":919,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:04.422275+0000","last_change":"2024-09-18T00:57:54.170233+0000","last_active":"2024-09-18T01:35:04.422275+0000","last_peered":"2024-09-18T01:35:04.422275+0000","last_clean":"2024-09-18T01:35:04.422275+0000","last_became_active":"2024-09-18T00:57:52.159283+0000","last_became_peered":"2024-09-18T00:57:52.159283+0000","last_unstale":"2024-09-18T01:35:04.422275+0000","last_undegraded":"2024-09-18T01:35:04.422275+0000","last_fullsized":"2024-09-18T01:35:04.422275+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:56:30.005098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00113901,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":914,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:04.545843+0000","last_change":"2024-09-18T00:57:54.167662+0000","last_active":"2024-09-18T01:35:04.545843+0000","last_peered":"2024-09-18T01:35:04.545843+0000","last_clean":"2024-09-18T01:35:04.545843+0000","last_became_active":"2024-09-18T00:57:52.160105+0000","last_became_peered":"2024-09-18T00:57:52.160105+0000","last_unstale":"2024-09-18T01:35:04.545843+0000","last_undegraded":"2024-09-18T01:35:04.545843+0000","last_fullsized":"2024-09-18T01:35:04.545843+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:10:35.692324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041490800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":915,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:04.546330+0000","last_change":"2024-09-18T00:57:54.167698+0000","last_active":"2024-09-18T01:35:04.546330+0000","last_peered":"2024-09-18T01:35:04.546330+0000","last_clean":"2024-09-18T01:35:04.546330+0000","last_became_active":"2024-09-18T00:57:52.159792+0000","last_became_peered":"2024-09-18T00:57:52.159792+0000","last_unstale":"2024-09-18T01:35:04.546330+0000","last_undegraded":"2024-09-18T01:35:04.546330+0000","last_fullsized":"2024-09-18T01:35:04.546330+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:42:59.227183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045536600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":916,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:04.149266+0000","last_change":"2024-09-18T00:57:54.172073+0000","last_active":"2024-09-18T01:35:04.149266+0000","last_peered":"2024-09-18T01:35:04.149266+0000","last_clean":"2024-09-18T01:35:04.149266+0000","last_became_active":"2024-09-18T00:57:52.158758+0000","last_became_peered":"2024-09-18T00:57:52.158758+0000","last_unstale":"2024-09-18T01:35:04.149266+0000","last_undegraded":"2024-09-18T01:35:04.149266+0000","last_fullsized":"2024-09-18T01:35:04.149266+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:51.138646+0000","last_clean_scrub_stamp":"2024-09-18T00:57:51.138646+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:44:25.148787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053750599999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'97","reported_seq":1044,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:04.148191+0000","last_change":"2024-09-18T00:57:47.137803+0000","last_active":"2024-09-18T01:35:04.148191+0000","last_peered":"2024-09-18T01:35:04.148191+0000","last_clean":"2024-09-18T01:35:04.148191+0000","last_became_active":"2024-09-18T00:57:47.136874+0000","last_became_peered":"2024-09-18T00:57:47.136874+0000","last_unstale":"2024-09-18T01:35:04.148191+0000","last_undegraded":"2024-09-18T01:35:04.148191+0000","last_fullsized":"2024-09-18T01:35:04.148191+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:46.126096+0000","last_clean_scrub_stamp":"2024-09-18T00:57:46.126096+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T04:09:27.863635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706116,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1427132,"kb_used_data":5284,"kb_used_omap":1,"kb_used_meta":1421822,"kb_avail":92944708,"statfs":{"total":96636764160,"available":95175380992,"internally_reserved":0,"allocated":5410816,"data_stored":2650932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1455946181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738819,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1377588,"kb_used_data":5344,"kb_used_omap":1,"kb_used_meta":1372222,"kb_avail":92994252,"statfs":{"total":96636764160,"available":95226114048,"internally_reserved":0,"allocated":5472256,"data_stored":3241319,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":1405155781},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738821,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1694644,"kb_used_data":5344,"kb_used_omap":1,"kb_used_meta":1689278,"kb_avail":92677196,"statfs":{"total":96636764160,"available":94901448704,"internally_reserved":0,"allocated":5472256,"data_stored":3241319,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1599,"internal_metadata":1729821121},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:35:09.087 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-18T00:57:51.138646+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=51, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:09.088 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-18T00:57:51.138646+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=51, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:09.088 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-18T00:57:51.138646+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=51, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:09.088 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-18T00:57:51.138646+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=51, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:09.088 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-18T00:57:51.138646+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=51, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:09.088 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-18T00:57:51.138646+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=51, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:09.088 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-18T00:57:51.138646+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=51, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:09.088 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-18T00:57:51.138646+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=51, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:09.088 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-18T00:57:46.126096+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=46, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=6, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:09.089 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-18T01:35:29.090 DEBUG:teuthology.orchestra.run.smithi150:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-09-18T01:35:29.377 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-09-18T01:35:29.378 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-09-18T01:35:29.390 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":1586,"stamp":"2024-09-18T01:35:27.638396+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4499348,"kb_used_data":15972,"kb_used_omap":4,"kb_used_meta":4483323,"kb_avail":278616172,"statfs":{"total":289910292480,"available":285302960128,"internally_reserved":0,"allocated":16355328,"data_stored":9133570,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4789,"internal_metadata":4590923083},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002839"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":924,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:13.887163+0000","last_change":"2024-09-18T01:35:13.887202+0000","last_active":"2024-09-18T01:35:13.887163+0000","last_peered":"2024-09-18T01:35:13.887163+0000","last_clean":"2024-09-18T01:35:13.887163+0000","last_became_active":"2024-09-18T00:57:52.161899+0000","last_became_peered":"2024-09-18T00:57:52.161899+0000","last_unstale":"2024-09-18T01:35:13.887163+0000","last_undegraded":"2024-09-18T01:35:13.887163+0000","last_fullsized":"2024-09-18T01:35:13.887163+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:13.887054+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:13.887054+0000","last_clean_scrub_stamp":"2024-09-18T01:35:13.887054+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T11:49:09.789766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075298400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":924,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:10.962347+0000","last_change":"2024-09-18T01:35:10.962369+0000","last_active":"2024-09-18T01:35:10.962347+0000","last_peered":"2024-09-18T01:35:10.962347+0000","last_clean":"2024-09-18T01:35:10.962347+0000","last_became_active":"2024-09-18T00:57:52.159679+0000","last_became_peered":"2024-09-18T00:57:52.159679+0000","last_unstale":"2024-09-18T01:35:10.962347+0000","last_undegraded":"2024-09-18T01:35:10.962347+0000","last_fullsized":"2024-09-18T01:35:10.962347+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:10.962232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:10.962232+0000","last_clean_scrub_stamp":"2024-09-18T01:35:10.962232+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:16:43.708135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054889900000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":924,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:11.961325+0000","last_change":"2024-09-18T01:35:11.961348+0000","last_active":"2024-09-18T01:35:11.961325+0000","last_peered":"2024-09-18T01:35:11.961325+0000","last_clean":"2024-09-18T01:35:11.961325+0000","last_became_active":"2024-09-18T00:57:52.160197+0000","last_became_peered":"2024-09-18T00:57:52.160197+0000","last_unstale":"2024-09-18T01:35:11.961325+0000","last_undegraded":"2024-09-18T01:35:11.961325+0000","last_fullsized":"2024-09-18T01:35:11.961325+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:11.961230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:11.961230+0000","last_clean_scrub_stamp":"2024-09-18T01:35:11.961230+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:39:39.309691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096762200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":925,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:12.913386+0000","last_change":"2024-09-18T01:35:12.913386+0000","last_active":"2024-09-18T01:35:12.913386+0000","last_peered":"2024-09-18T01:35:12.913386+0000","last_clean":"2024-09-18T01:35:12.913386+0000","last_became_active":"2024-09-18T00:57:52.159800+0000","last_became_peered":"2024-09-18T00:57:52.159800+0000","last_unstale":"2024-09-18T01:35:12.913386+0000","last_undegraded":"2024-09-18T01:35:12.913386+0000","last_fullsized":"2024-09-18T01:35:12.913386+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:12.913263+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:12.913263+0000","last_clean_scrub_stamp":"2024-09-18T01:35:12.913263+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T04:57:07.969624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00074951200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"457'3","reported_seq":930,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:08.378143+0000","last_change":"2024-09-18T01:35:08.378143+0000","last_active":"2024-09-18T01:35:08.378143+0000","last_peered":"2024-09-18T01:35:08.378143+0000","last_clean":"2024-09-18T01:35:08.378143+0000","last_became_active":"2024-09-18T00:57:52.159283+0000","last_became_peered":"2024-09-18T00:57:52.159283+0000","last_unstale":"2024-09-18T01:35:08.378143+0000","last_undegraded":"2024-09-18T01:35:08.378143+0000","last_fullsized":"2024-09-18T01:35:08.378143+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"457'3","last_scrub_stamp":"2024-09-18T01:35:08.377804+0000","last_deep_scrub":"457'3","last_deep_scrub_stamp":"2024-09-18T01:35:08.377804+0000","last_clean_scrub_stamp":"2024-09-18T01:35:08.377804+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T10:58:24.235769+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00113901,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":924,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:10.573566+0000","last_change":"2024-09-18T01:35:10.573566+0000","last_active":"2024-09-18T01:35:10.573566+0000","last_peered":"2024-09-18T01:35:10.573566+0000","last_clean":"2024-09-18T01:35:10.573566+0000","last_became_active":"2024-09-18T00:57:52.160105+0000","last_became_peered":"2024-09-18T00:57:52.160105+0000","last_unstale":"2024-09-18T01:35:10.573566+0000","last_undegraded":"2024-09-18T01:35:10.573566+0000","last_fullsized":"2024-09-18T01:35:10.573566+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:10.573380+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:10.573380+0000","last_clean_scrub_stamp":"2024-09-18T01:35:10.573380+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T12:28:00.673470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041490800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":924,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:09.582243+0000","last_change":"2024-09-18T01:35:09.582265+0000","last_active":"2024-09-18T01:35:09.582243+0000","last_peered":"2024-09-18T01:35:09.582243+0000","last_clean":"2024-09-18T01:35:09.582243+0000","last_became_active":"2024-09-18T00:57:52.159792+0000","last_became_peered":"2024-09-18T00:57:52.159792+0000","last_unstale":"2024-09-18T01:35:09.582243+0000","last_undegraded":"2024-09-18T01:35:09.582243+0000","last_fullsized":"2024-09-18T01:35:09.582243+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:09.582165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:09.582165+0000","last_clean_scrub_stamp":"2024-09-18T01:35:09.582165+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:19:58.563040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045536600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"457'2","reported_seq":927,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:09.929588+0000","last_change":"2024-09-18T01:35:09.929588+0000","last_active":"2024-09-18T01:35:09.929588+0000","last_peered":"2024-09-18T01:35:09.929588+0000","last_clean":"2024-09-18T01:35:09.929588+0000","last_became_active":"2024-09-18T00:57:52.158758+0000","last_became_peered":"2024-09-18T00:57:52.158758+0000","last_unstale":"2024-09-18T01:35:09.929588+0000","last_undegraded":"2024-09-18T01:35:09.929588+0000","last_fullsized":"2024-09-18T01:35:09.929588+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"457'2","last_scrub_stamp":"2024-09-18T01:35:09.929471+0000","last_deep_scrub":"457'2","last_deep_scrub_stamp":"2024-09-18T01:35:09.929471+0000","last_clean_scrub_stamp":"2024-09-18T01:35:09.929471+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T12:03:47.660914+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00053750599999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"457'99","reported_seq":1055,"reported_epoch":457,"state":"active+clean","last_fresh":"2024-09-18T01:35:08.974339+0000","last_change":"2024-09-18T01:35:08.974395+0000","last_active":"2024-09-18T01:35:08.974339+0000","last_peered":"2024-09-18T01:35:08.974339+0000","last_clean":"2024-09-18T01:35:08.974339+0000","last_became_active":"2024-09-18T00:57:47.136874+0000","last_became_peered":"2024-09-18T00:57:47.136874+0000","last_unstale":"2024-09-18T01:35:08.974339+0000","last_undegraded":"2024-09-18T01:35:08.974339+0000","last_fullsized":"2024-09-18T01:35:08.974339+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"457'99","last_scrub_stamp":"2024-09-18T01:35:08.974257+0000","last_deep_scrub":"457'99","last_deep_scrub_stamp":"2024-09-18T01:35:08.974257+0000","last_clean_scrub_stamp":"2024-09-18T01:35:08.974257+0000","objects_scrubbed":2,"log_size":99,"log_dups_size":0,"ondisk_log_size":99,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:24:18.066233+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11642,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":99,"ondisk_log_size":99,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706120,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1427124,"kb_used_data":5284,"kb_used_omap":1,"kb_used_meta":1421822,"kb_avail":92944716,"statfs":{"total":96636764160,"available":95175389184,"internally_reserved":0,"allocated":5410816,"data_stored":2650932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":1455946181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738823,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1377584,"kb_used_data":5344,"kb_used_omap":1,"kb_used_meta":1372222,"kb_avail":92994256,"statfs":{"total":96636764160,"available":95226118144,"internally_reserved":0,"allocated":5472256,"data_stored":3241319,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":1405155781},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738825,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1694640,"kb_used_data":5344,"kb_used_omap":1,"kb_used_meta":1689278,"kb_avail":92677200,"statfs":{"total":96636764160,"available":94901452800,"internally_reserved":0,"allocated":5472256,"data_stored":3241319,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1599,"internal_metadata":1729821121},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5821,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:35:29.391 DEBUG:teuthology.orchestra.run.smithi150:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2024-09-18T01:35:29.683 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-18T01:35:29.683 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-18T01:35:29.683 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2024-09-18T01:35:29.683 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:29.757 INFO:tasks.ceph.cluster2.osd.0:Stopped 2024-09-18T01:35:29.757 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2024-09-18T01:35:29.758 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:29.830 INFO:tasks.ceph.cluster2.osd.1:Stopped 2024-09-18T01:35:29.830 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2024-09-18T01:35:29.830 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:29.916 INFO:tasks.ceph.cluster2.osd.2:Stopped 2024-09-18T01:35:29.916 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-18T01:35:29.916 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2024-09-18T01:35:29.916 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:29.945 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2024-09-18T01:35:29.945 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-18T01:35:29.945 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2024-09-18T01:35:29.945 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:29.958 INFO:tasks.ceph.cluster2.mon.a:Stopped 2024-09-18T01:35:29.958 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-18T01:35:29.958 DEBUG:teuthology.orchestra.run.smithi150:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2024-09-18T01:35:30.084 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@smithi150.front.sepia.ceph.com 2024-09-18T01:35:30.085 DEBUG:teuthology.orchestra.run.smithi150:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2024-09-18T01:35:30.425 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@smithi150.front.sepia.ceph.com 2024-09-18T01:35:30.425 DEBUG:teuthology.orchestra.run.smithi150:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2024-09-18T01:35:30.568 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@smithi150.front.sepia.ceph.com 2024-09-18T01:35:30.569 DEBUG:teuthology.orchestra.run.smithi150:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2024-09-18T01:35:30.739 INFO:tasks.ceph:Archiving mon data... 2024-09-18T01:35:30.740 DEBUG:teuthology.misc:Transferring archived files from smithi150:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686/data/cluster2.mon.a.tgz 2024-09-18T01:35:30.741 DEBUG:teuthology.orchestra.run.smithi150:> mktemp 2024-09-18T01:35:30.757 INFO:teuthology.orchestra.run.smithi150.stdout:/tmp/tmp.jFsdqxz2TK 2024-09-18T01:35:30.757 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.jFsdqxz2TK 2024-09-18T01:35:30.943 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 0666 /tmp/tmp.jFsdqxz2TK 2024-09-18T01:35:31.072 DEBUG:teuthology.orchestra.remote:smithi150:/tmp/tmp.jFsdqxz2TK is 496KB 2024-09-18T01:35:31.143 DEBUG:teuthology.orchestra.run.smithi150:> rm -fr /tmp/tmp.jFsdqxz2TK 2024-09-18T01:35:31.158 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-18T01:35:31.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2024-09-18T01:35:31.162 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2024-09-18T01:35:31.225 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-18T01:35:31.245 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2024-09-18T01:35:31.245 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T01:35:31.526 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T01:35:31.526 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-18T01:35:31.538 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1614,"stamp":"2024-09-18T01:35:30.353830+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":271332,"kb_used_data":11304,"kb_used_omap":4,"kb_used_meta":259963,"kb_avail":282844188,"statfs":{"total":289910292480,"available":289632448512,"internally_reserved":0,"allocated":11575296,"data_stored":7295605,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":266202447},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002534"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":930,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.956683+0000","last_change":"2024-09-18T00:57:15.827787+0000","last_active":"2024-09-18T01:35:02.956683+0000","last_peered":"2024-09-18T01:35:02.956683+0000","last_clean":"2024-09-18T01:35:02.956683+0000","last_became_active":"2024-09-18T00:57:13.815654+0000","last_became_peered":"2024-09-18T00:57:13.815654+0000","last_unstale":"2024-09-18T01:35:02.956683+0000","last_undegraded":"2024-09-18T01:35:02.956683+0000","last_fullsized":"2024-09-18T01:35:02.956683+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:31.448857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088783099999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":930,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.958117+0000","last_change":"2024-09-18T00:57:15.827523+0000","last_active":"2024-09-18T01:35:02.958117+0000","last_peered":"2024-09-18T01:35:02.958117+0000","last_clean":"2024-09-18T01:35:02.958117+0000","last_became_active":"2024-09-18T00:57:13.817123+0000","last_became_peered":"2024-09-18T00:57:13.817123+0000","last_unstale":"2024-09-18T01:35:02.958117+0000","last_undegraded":"2024-09-18T01:35:02.958117+0000","last_fullsized":"2024-09-18T01:35:02.958117+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:30.781530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000818183,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.957680+0000","last_change":"2024-09-18T00:57:15.827552+0000","last_active":"2024-09-18T01:35:02.957680+0000","last_peered":"2024-09-18T01:35:02.957680+0000","last_clean":"2024-09-18T01:35:02.957680+0000","last_became_active":"2024-09-18T00:57:13.816435+0000","last_became_peered":"2024-09-18T00:57:13.816435+0000","last_unstale":"2024-09-18T01:35:02.957680+0000","last_undegraded":"2024-09-18T01:35:02.957680+0000","last_fullsized":"2024-09-18T01:35:02.957680+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T11:59:58.580873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086916700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.955974+0000","last_change":"2024-09-18T00:57:15.827187+0000","last_active":"2024-09-18T01:35:02.955974+0000","last_peered":"2024-09-18T01:35:02.955974+0000","last_clean":"2024-09-18T01:35:02.955974+0000","last_became_active":"2024-09-18T00:57:13.815768+0000","last_became_peered":"2024-09-18T00:57:13.815768+0000","last_unstale":"2024-09-18T01:35:02.955974+0000","last_undegraded":"2024-09-18T01:35:02.955974+0000","last_fullsized":"2024-09-18T01:35:02.955974+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:21:50.803048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000610718,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.798631+0000","last_change":"2024-09-18T00:57:15.825320+0000","last_active":"2024-09-18T01:35:03.798631+0000","last_peered":"2024-09-18T01:35:03.798631+0000","last_clean":"2024-09-18T01:35:03.798631+0000","last_became_active":"2024-09-18T00:57:13.812378+0000","last_became_peered":"2024-09-18T00:57:13.812378+0000","last_unstale":"2024-09-18T01:35:03.798631+0000","last_undegraded":"2024-09-18T01:35:03.798631+0000","last_fullsized":"2024-09-18T01:35:03.798631+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:51:57.842731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001073121,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.800121+0000","last_change":"2024-09-18T00:57:15.822211+0000","last_active":"2024-09-18T01:35:03.800121+0000","last_peered":"2024-09-18T01:35:03.800121+0000","last_clean":"2024-09-18T01:35:03.800121+0000","last_became_active":"2024-09-18T00:57:13.815977+0000","last_became_peered":"2024-09-18T00:57:13.815977+0000","last_unstale":"2024-09-18T01:35:03.800121+0000","last_undegraded":"2024-09-18T01:35:03.800121+0000","last_fullsized":"2024-09-18T01:35:03.800121+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:30:33.665761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040989599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":932,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.801231+0000","last_change":"2024-09-18T00:57:15.822309+0000","last_active":"2024-09-18T01:35:03.801231+0000","last_peered":"2024-09-18T01:35:03.801231+0000","last_clean":"2024-09-18T01:35:03.801231+0000","last_became_active":"2024-09-18T00:57:13.816220+0000","last_became_peered":"2024-09-18T00:57:13.816220+0000","last_unstale":"2024-09-18T01:35:03.801231+0000","last_undegraded":"2024-09-18T01:35:03.801231+0000","last_fullsized":"2024-09-18T01:35:03.801231+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:28:51.286449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049982200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":932,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.958777+0000","last_change":"2024-09-18T00:57:15.827239+0000","last_active":"2024-09-18T01:35:02.958777+0000","last_peered":"2024-09-18T01:35:02.958777+0000","last_clean":"2024-09-18T01:35:02.958777+0000","last_became_active":"2024-09-18T00:57:13.815577+0000","last_became_peered":"2024-09-18T00:57:13.815577+0000","last_unstale":"2024-09-18T01:35:02.958777+0000","last_undegraded":"2024-09-18T01:35:02.958777+0000","last_fullsized":"2024-09-18T01:35:02.958777+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:42:24.720766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066113999999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'97","reported_seq":1060,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.960946+0000","last_change":"2024-09-18T00:57:09.799222+0000","last_active":"2024-09-18T01:35:02.960946+0000","last_peered":"2024-09-18T01:35:02.960946+0000","last_clean":"2024-09-18T01:35:02.960946+0000","last_became_active":"2024-09-18T00:57:09.796934+0000","last_became_peered":"2024-09-18T00:57:09.796934+0000","last_unstale":"2024-09-18T01:35:02.960946+0000","last_undegraded":"2024-09-18T01:35:02.960946+0000","last_fullsized":"2024-09-18T01:35:02.960946+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_clean_scrub_stamp":"2024-09-18T00:57:08.784496+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:37:26.386956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706128,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":88996,"kb_used_data":3728,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94282844,"statfs":{"total":96636764160,"available":96545632256,"internally_reserved":0,"allocated":3817472,"data_stored":2038277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":87292357},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706127,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":115424,"kb_used_data":3788,"kb_used_omap":1,"kb_used_meta":111614,"kb_avail":94256416,"statfs":{"total":96636764160,"available":96518569984,"internally_reserved":0,"allocated":3878912,"data_stored":2628664,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":114293189},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738832,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66912,"kb_used_data":3788,"kb_used_omap":1,"kb_used_meta":63102,"kb_avail":94304928,"statfs":{"total":96636764160,"available":96568246272,"internally_reserved":0,"allocated":3878912,"data_stored":2628664,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":64616901},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:35:31.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T01:35:31.810 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T01:35:31.811 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-18T01:35:31.823 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1614,"stamp":"2024-09-18T01:35:30.353830+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":271332,"kb_used_data":11304,"kb_used_omap":4,"kb_used_meta":259963,"kb_avail":282844188,"statfs":{"total":289910292480,"available":289632448512,"internally_reserved":0,"allocated":11575296,"data_stored":7295605,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":266202447},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002534"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":930,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.956683+0000","last_change":"2024-09-18T00:57:15.827787+0000","last_active":"2024-09-18T01:35:02.956683+0000","last_peered":"2024-09-18T01:35:02.956683+0000","last_clean":"2024-09-18T01:35:02.956683+0000","last_became_active":"2024-09-18T00:57:13.815654+0000","last_became_peered":"2024-09-18T00:57:13.815654+0000","last_unstale":"2024-09-18T01:35:02.956683+0000","last_undegraded":"2024-09-18T01:35:02.956683+0000","last_fullsized":"2024-09-18T01:35:02.956683+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:31.448857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088783099999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":930,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.958117+0000","last_change":"2024-09-18T00:57:15.827523+0000","last_active":"2024-09-18T01:35:02.958117+0000","last_peered":"2024-09-18T01:35:02.958117+0000","last_clean":"2024-09-18T01:35:02.958117+0000","last_became_active":"2024-09-18T00:57:13.817123+0000","last_became_peered":"2024-09-18T00:57:13.817123+0000","last_unstale":"2024-09-18T01:35:02.958117+0000","last_undegraded":"2024-09-18T01:35:02.958117+0000","last_fullsized":"2024-09-18T01:35:02.958117+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:30.781530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000818183,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.957680+0000","last_change":"2024-09-18T00:57:15.827552+0000","last_active":"2024-09-18T01:35:02.957680+0000","last_peered":"2024-09-18T01:35:02.957680+0000","last_clean":"2024-09-18T01:35:02.957680+0000","last_became_active":"2024-09-18T00:57:13.816435+0000","last_became_peered":"2024-09-18T00:57:13.816435+0000","last_unstale":"2024-09-18T01:35:02.957680+0000","last_undegraded":"2024-09-18T01:35:02.957680+0000","last_fullsized":"2024-09-18T01:35:02.957680+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T11:59:58.580873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086916700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.955974+0000","last_change":"2024-09-18T00:57:15.827187+0000","last_active":"2024-09-18T01:35:02.955974+0000","last_peered":"2024-09-18T01:35:02.955974+0000","last_clean":"2024-09-18T01:35:02.955974+0000","last_became_active":"2024-09-18T00:57:13.815768+0000","last_became_peered":"2024-09-18T00:57:13.815768+0000","last_unstale":"2024-09-18T01:35:02.955974+0000","last_undegraded":"2024-09-18T01:35:02.955974+0000","last_fullsized":"2024-09-18T01:35:02.955974+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:21:50.803048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000610718,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.798631+0000","last_change":"2024-09-18T00:57:15.825320+0000","last_active":"2024-09-18T01:35:03.798631+0000","last_peered":"2024-09-18T01:35:03.798631+0000","last_clean":"2024-09-18T01:35:03.798631+0000","last_became_active":"2024-09-18T00:57:13.812378+0000","last_became_peered":"2024-09-18T00:57:13.812378+0000","last_unstale":"2024-09-18T01:35:03.798631+0000","last_undegraded":"2024-09-18T01:35:03.798631+0000","last_fullsized":"2024-09-18T01:35:03.798631+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:51:57.842731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001073121,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.800121+0000","last_change":"2024-09-18T00:57:15.822211+0000","last_active":"2024-09-18T01:35:03.800121+0000","last_peered":"2024-09-18T01:35:03.800121+0000","last_clean":"2024-09-18T01:35:03.800121+0000","last_became_active":"2024-09-18T00:57:13.815977+0000","last_became_peered":"2024-09-18T00:57:13.815977+0000","last_unstale":"2024-09-18T01:35:03.800121+0000","last_undegraded":"2024-09-18T01:35:03.800121+0000","last_fullsized":"2024-09-18T01:35:03.800121+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:30:33.665761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040989599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":932,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.801231+0000","last_change":"2024-09-18T00:57:15.822309+0000","last_active":"2024-09-18T01:35:03.801231+0000","last_peered":"2024-09-18T01:35:03.801231+0000","last_clean":"2024-09-18T01:35:03.801231+0000","last_became_active":"2024-09-18T00:57:13.816220+0000","last_became_peered":"2024-09-18T00:57:13.816220+0000","last_unstale":"2024-09-18T01:35:03.801231+0000","last_undegraded":"2024-09-18T01:35:03.801231+0000","last_fullsized":"2024-09-18T01:35:03.801231+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:28:51.286449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049982200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":932,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.958777+0000","last_change":"2024-09-18T00:57:15.827239+0000","last_active":"2024-09-18T01:35:02.958777+0000","last_peered":"2024-09-18T01:35:02.958777+0000","last_clean":"2024-09-18T01:35:02.958777+0000","last_became_active":"2024-09-18T00:57:13.815577+0000","last_became_peered":"2024-09-18T00:57:13.815577+0000","last_unstale":"2024-09-18T01:35:02.958777+0000","last_undegraded":"2024-09-18T01:35:02.958777+0000","last_fullsized":"2024-09-18T01:35:02.958777+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:42:24.720766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066113999999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'97","reported_seq":1060,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.960946+0000","last_change":"2024-09-18T00:57:09.799222+0000","last_active":"2024-09-18T01:35:02.960946+0000","last_peered":"2024-09-18T01:35:02.960946+0000","last_clean":"2024-09-18T01:35:02.960946+0000","last_became_active":"2024-09-18T00:57:09.796934+0000","last_became_peered":"2024-09-18T00:57:09.796934+0000","last_unstale":"2024-09-18T01:35:02.960946+0000","last_undegraded":"2024-09-18T01:35:02.960946+0000","last_fullsized":"2024-09-18T01:35:02.960946+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_clean_scrub_stamp":"2024-09-18T00:57:08.784496+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:37:26.386956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706128,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":88996,"kb_used_data":3728,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94282844,"statfs":{"total":96636764160,"available":96545632256,"internally_reserved":0,"allocated":3817472,"data_stored":2038277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":87292357},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706127,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":115424,"kb_used_data":3788,"kb_used_omap":1,"kb_used_meta":111614,"kb_avail":94256416,"statfs":{"total":96636764160,"available":96518569984,"internally_reserved":0,"allocated":3878912,"data_stored":2628664,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":114293189},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738832,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66912,"kb_used_data":3788,"kb_used_omap":1,"kb_used_meta":63102,"kb_avail":94304928,"statfs":{"total":96636764160,"available":96568246272,"internally_reserved":0,"allocated":3878912,"data_stored":2628664,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":64616901},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:35:31.824 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2024-09-18T01:35:31.824 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T01:35:32.097 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T01:35:32.097 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-18T01:35:32.109 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1614,"stamp":"2024-09-18T01:35:30.353830+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":271332,"kb_used_data":11304,"kb_used_omap":4,"kb_used_meta":259963,"kb_avail":282844188,"statfs":{"total":289910292480,"available":289632448512,"internally_reserved":0,"allocated":11575296,"data_stored":7295605,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":266202447},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002534"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":930,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.956683+0000","last_change":"2024-09-18T00:57:15.827787+0000","last_active":"2024-09-18T01:35:02.956683+0000","last_peered":"2024-09-18T01:35:02.956683+0000","last_clean":"2024-09-18T01:35:02.956683+0000","last_became_active":"2024-09-18T00:57:13.815654+0000","last_became_peered":"2024-09-18T00:57:13.815654+0000","last_unstale":"2024-09-18T01:35:02.956683+0000","last_undegraded":"2024-09-18T01:35:02.956683+0000","last_fullsized":"2024-09-18T01:35:02.956683+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:31.448857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088783099999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":930,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.958117+0000","last_change":"2024-09-18T00:57:15.827523+0000","last_active":"2024-09-18T01:35:02.958117+0000","last_peered":"2024-09-18T01:35:02.958117+0000","last_clean":"2024-09-18T01:35:02.958117+0000","last_became_active":"2024-09-18T00:57:13.817123+0000","last_became_peered":"2024-09-18T00:57:13.817123+0000","last_unstale":"2024-09-18T01:35:02.958117+0000","last_undegraded":"2024-09-18T01:35:02.958117+0000","last_fullsized":"2024-09-18T01:35:02.958117+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:30.781530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000818183,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.957680+0000","last_change":"2024-09-18T00:57:15.827552+0000","last_active":"2024-09-18T01:35:02.957680+0000","last_peered":"2024-09-18T01:35:02.957680+0000","last_clean":"2024-09-18T01:35:02.957680+0000","last_became_active":"2024-09-18T00:57:13.816435+0000","last_became_peered":"2024-09-18T00:57:13.816435+0000","last_unstale":"2024-09-18T01:35:02.957680+0000","last_undegraded":"2024-09-18T01:35:02.957680+0000","last_fullsized":"2024-09-18T01:35:02.957680+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T11:59:58.580873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086916700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.955974+0000","last_change":"2024-09-18T00:57:15.827187+0000","last_active":"2024-09-18T01:35:02.955974+0000","last_peered":"2024-09-18T01:35:02.955974+0000","last_clean":"2024-09-18T01:35:02.955974+0000","last_became_active":"2024-09-18T00:57:13.815768+0000","last_became_peered":"2024-09-18T00:57:13.815768+0000","last_unstale":"2024-09-18T01:35:02.955974+0000","last_undegraded":"2024-09-18T01:35:02.955974+0000","last_fullsized":"2024-09-18T01:35:02.955974+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:21:50.803048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000610718,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.798631+0000","last_change":"2024-09-18T00:57:15.825320+0000","last_active":"2024-09-18T01:35:03.798631+0000","last_peered":"2024-09-18T01:35:03.798631+0000","last_clean":"2024-09-18T01:35:03.798631+0000","last_became_active":"2024-09-18T00:57:13.812378+0000","last_became_peered":"2024-09-18T00:57:13.812378+0000","last_unstale":"2024-09-18T01:35:03.798631+0000","last_undegraded":"2024-09-18T01:35:03.798631+0000","last_fullsized":"2024-09-18T01:35:03.798631+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:51:57.842731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001073121,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.800121+0000","last_change":"2024-09-18T00:57:15.822211+0000","last_active":"2024-09-18T01:35:03.800121+0000","last_peered":"2024-09-18T01:35:03.800121+0000","last_clean":"2024-09-18T01:35:03.800121+0000","last_became_active":"2024-09-18T00:57:13.815977+0000","last_became_peered":"2024-09-18T00:57:13.815977+0000","last_unstale":"2024-09-18T01:35:03.800121+0000","last_undegraded":"2024-09-18T01:35:03.800121+0000","last_fullsized":"2024-09-18T01:35:03.800121+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:30:33.665761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040989599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":932,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.801231+0000","last_change":"2024-09-18T00:57:15.822309+0000","last_active":"2024-09-18T01:35:03.801231+0000","last_peered":"2024-09-18T01:35:03.801231+0000","last_clean":"2024-09-18T01:35:03.801231+0000","last_became_active":"2024-09-18T00:57:13.816220+0000","last_became_peered":"2024-09-18T00:57:13.816220+0000","last_unstale":"2024-09-18T01:35:03.801231+0000","last_undegraded":"2024-09-18T01:35:03.801231+0000","last_fullsized":"2024-09-18T01:35:03.801231+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:28:51.286449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049982200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":932,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.958777+0000","last_change":"2024-09-18T00:57:15.827239+0000","last_active":"2024-09-18T01:35:02.958777+0000","last_peered":"2024-09-18T01:35:02.958777+0000","last_clean":"2024-09-18T01:35:02.958777+0000","last_became_active":"2024-09-18T00:57:13.815577+0000","last_became_peered":"2024-09-18T00:57:13.815577+0000","last_unstale":"2024-09-18T01:35:02.958777+0000","last_undegraded":"2024-09-18T01:35:02.958777+0000","last_fullsized":"2024-09-18T01:35:02.958777+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:42:24.720766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066113999999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'97","reported_seq":1060,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.960946+0000","last_change":"2024-09-18T00:57:09.799222+0000","last_active":"2024-09-18T01:35:02.960946+0000","last_peered":"2024-09-18T01:35:02.960946+0000","last_clean":"2024-09-18T01:35:02.960946+0000","last_became_active":"2024-09-18T00:57:09.796934+0000","last_became_peered":"2024-09-18T00:57:09.796934+0000","last_unstale":"2024-09-18T01:35:02.960946+0000","last_undegraded":"2024-09-18T01:35:02.960946+0000","last_fullsized":"2024-09-18T01:35:02.960946+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_clean_scrub_stamp":"2024-09-18T00:57:08.784496+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:37:26.386956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706128,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":88996,"kb_used_data":3728,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94282844,"statfs":{"total":96636764160,"available":96545632256,"internally_reserved":0,"allocated":3817472,"data_stored":2038277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":87292357},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706127,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":115424,"kb_used_data":3788,"kb_used_omap":1,"kb_used_meta":111614,"kb_avail":94256416,"statfs":{"total":96636764160,"available":96518569984,"internally_reserved":0,"allocated":3878912,"data_stored":2628664,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":114293189},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738832,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66912,"kb_used_data":3788,"kb_used_omap":1,"kb_used_meta":63102,"kb_avail":94304928,"statfs":{"total":96636764160,"available":96568246272,"internally_reserved":0,"allocated":3878912,"data_stored":2628664,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":64616901},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:35:32.110 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-09-18T01:35:32.385 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T01:35:32.385 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":464,"fsid":"539007ae-4341-41e5-b60c-c3d4ca69948f","created":"2024-09-18T00:56:57.533867+0000","modified":"2024-09-18T01:35:02.946949+0000","last_up_change":"2024-09-18T00:57:06.780761+0000","last_in_change":"2024-09-18T00:56:58.454372+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-18T00:57:07.803521+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-18T00:57:12.626407+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"812a085a-d9f9-4f18-944b-7ac74fa0c228","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":30,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6809","nonce":814730911}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6811","nonce":814730911}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6815","nonce":814730911}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":814730911},{"type":"v1","addr":"172.21.15.78:6813","nonce":814730911}]},"public_addr":"172.21.15.78:6809/814730911","cluster_addr":"172.21.15.78:6811/814730911","heartbeat_back_addr":"172.21.15.78:6815/814730911","heartbeat_front_addr":"172.21.15.78:6813/814730911","state":["exists","up"]},{"osd":1,"uuid":"66a760e0-5971-4cbe-a451-985e9d8dd97f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":30,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6801","nonce":3340322889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6803","nonce":3340322889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6807","nonce":3340322889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":3340322889},{"type":"v1","addr":"172.21.15.78:6805","nonce":3340322889}]},"public_addr":"172.21.15.78:6801/3340322889","cluster_addr":"172.21.15.78:6803/3340322889","heartbeat_back_addr":"172.21.15.78:6807/3340322889","heartbeat_front_addr":"172.21.15.78:6805/3340322889","state":["exists","up"]},{"osd":2,"uuid":"bf096562-0f07-440d-aa1c-5ccb6f01b720","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":30,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6817","nonce":3900839217}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6819","nonce":3900839217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6823","nonce":3900839217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":3900839217},{"type":"v1","addr":"172.21.15.78:6821","nonce":3900839217}]},"public_addr":"172.21.15.78:6817/3900839217","cluster_addr":"172.21.15.78:6819/3900839217","heartbeat_back_addr":"172.21.15.78:6823/3900839217","heartbeat_front_addr":"172.21.15.78:6821/3900839217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:05.033037+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:04.521237+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-18T00:57:04.839863+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-18T01:35:33.399 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2024-09-18T01:35:33.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:35:33.531 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-18T01:35:33.531 INFO:teuthology.orchestra.run.smithi078.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:35:33.531 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-18T01:35:33.542 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2024-09-18T01:35:33.816 INFO:teuthology.orchestra.run.smithi078.stderr:instructed osd(s) 0 to deep-scrub 2024-09-18T01:35:33.830 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2024-09-18T01:35:33.830 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:35:33.952 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-18T01:35:33.952 INFO:teuthology.orchestra.run.smithi078.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:35:33.952 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-18T01:35:33.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2024-09-18T01:35:34.236 INFO:teuthology.orchestra.run.smithi078.stderr:instructed osd(s) 1 to deep-scrub 2024-09-18T01:35:34.249 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2024-09-18T01:35:34.249 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-18T01:35:34.370 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-18T01:35:34.371 INFO:teuthology.orchestra.run.smithi078.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2024-09-18T01:35:34.371 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-18T01:35:34.381 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2024-09-18T01:35:34.654 INFO:teuthology.orchestra.run.smithi078.stderr:instructed osd(s) 2 to deep-scrub 2024-09-18T01:35:34.667 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T01:35:34.941 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T01:35:34.941 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-18T01:35:34.953 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1616,"stamp":"2024-09-18T01:35:34.354397+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":271332,"kb_used_data":11304,"kb_used_omap":4,"kb_used_meta":259963,"kb_avail":282844188,"statfs":{"total":289910292480,"available":289632448512,"internally_reserved":0,"allocated":11575296,"data_stored":7295605,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":266202447},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002128"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":930,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.956683+0000","last_change":"2024-09-18T00:57:15.827787+0000","last_active":"2024-09-18T01:35:02.956683+0000","last_peered":"2024-09-18T01:35:02.956683+0000","last_clean":"2024-09-18T01:35:02.956683+0000","last_became_active":"2024-09-18T00:57:13.815654+0000","last_became_peered":"2024-09-18T00:57:13.815654+0000","last_unstale":"2024-09-18T01:35:02.956683+0000","last_undegraded":"2024-09-18T01:35:02.956683+0000","last_fullsized":"2024-09-18T01:35:02.956683+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:31.448857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088783099999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":930,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.958117+0000","last_change":"2024-09-18T00:57:15.827523+0000","last_active":"2024-09-18T01:35:02.958117+0000","last_peered":"2024-09-18T01:35:02.958117+0000","last_clean":"2024-09-18T01:35:02.958117+0000","last_became_active":"2024-09-18T00:57:13.817123+0000","last_became_peered":"2024-09-18T00:57:13.817123+0000","last_unstale":"2024-09-18T01:35:02.958117+0000","last_undegraded":"2024-09-18T01:35:02.958117+0000","last_fullsized":"2024-09-18T01:35:02.958117+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T02:37:30.781530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000818183,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.957680+0000","last_change":"2024-09-18T00:57:15.827552+0000","last_active":"2024-09-18T01:35:02.957680+0000","last_peered":"2024-09-18T01:35:02.957680+0000","last_clean":"2024-09-18T01:35:02.957680+0000","last_became_active":"2024-09-18T00:57:13.816435+0000","last_became_peered":"2024-09-18T00:57:13.816435+0000","last_unstale":"2024-09-18T01:35:02.957680+0000","last_undegraded":"2024-09-18T01:35:02.957680+0000","last_fullsized":"2024-09-18T01:35:02.957680+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T11:59:58.580873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086916700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.955974+0000","last_change":"2024-09-18T00:57:15.827187+0000","last_active":"2024-09-18T01:35:02.955974+0000","last_peered":"2024-09-18T01:35:02.955974+0000","last_clean":"2024-09-18T01:35:02.955974+0000","last_became_active":"2024-09-18T00:57:13.815768+0000","last_became_peered":"2024-09-18T00:57:13.815768+0000","last_unstale":"2024-09-18T01:35:02.955974+0000","last_undegraded":"2024-09-18T01:35:02.955974+0000","last_fullsized":"2024-09-18T01:35:02.955974+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:21:50.803048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000610718,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.798631+0000","last_change":"2024-09-18T00:57:15.825320+0000","last_active":"2024-09-18T01:35:03.798631+0000","last_peered":"2024-09-18T01:35:03.798631+0000","last_clean":"2024-09-18T01:35:03.798631+0000","last_became_active":"2024-09-18T00:57:13.812378+0000","last_became_peered":"2024-09-18T00:57:13.812378+0000","last_unstale":"2024-09-18T01:35:03.798631+0000","last_undegraded":"2024-09-18T01:35:03.798631+0000","last_fullsized":"2024-09-18T01:35:03.798631+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T07:51:57.842731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001073121,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":931,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.800121+0000","last_change":"2024-09-18T00:57:15.822211+0000","last_active":"2024-09-18T01:35:03.800121+0000","last_peered":"2024-09-18T01:35:03.800121+0000","last_clean":"2024-09-18T01:35:03.800121+0000","last_became_active":"2024-09-18T00:57:13.815977+0000","last_became_peered":"2024-09-18T00:57:13.815977+0000","last_unstale":"2024-09-18T01:35:03.800121+0000","last_undegraded":"2024-09-18T01:35:03.800121+0000","last_fullsized":"2024-09-18T01:35:03.800121+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:30:33.665761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040989599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":932,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:03.801231+0000","last_change":"2024-09-18T00:57:15.822309+0000","last_active":"2024-09-18T01:35:03.801231+0000","last_peered":"2024-09-18T01:35:03.801231+0000","last_clean":"2024-09-18T01:35:03.801231+0000","last_became_active":"2024-09-18T00:57:13.816220+0000","last_became_peered":"2024-09-18T00:57:13.816220+0000","last_unstale":"2024-09-18T01:35:03.801231+0000","last_undegraded":"2024-09-18T01:35:03.801231+0000","last_fullsized":"2024-09-18T01:35:03.801231+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T08:28:51.286449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049982200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":932,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.958777+0000","last_change":"2024-09-18T00:57:15.827239+0000","last_active":"2024-09-18T01:35:02.958777+0000","last_peered":"2024-09-18T01:35:02.958777+0000","last_clean":"2024-09-18T01:35:02.958777+0000","last_became_active":"2024-09-18T00:57:13.815577+0000","last_became_peered":"2024-09-18T00:57:13.815577+0000","last_unstale":"2024-09-18T01:35:02.958777+0000","last_undegraded":"2024-09-18T01:35:02.958777+0000","last_fullsized":"2024-09-18T01:35:02.958777+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:12.796466+0000","last_clean_scrub_stamp":"2024-09-18T00:57:12.796466+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:42:24.720766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066113999999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'97","reported_seq":1060,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:02.960946+0000","last_change":"2024-09-18T00:57:09.799222+0000","last_active":"2024-09-18T01:35:02.960946+0000","last_peered":"2024-09-18T01:35:02.960946+0000","last_clean":"2024-09-18T01:35:02.960946+0000","last_became_active":"2024-09-18T00:57:09.796934+0000","last_became_peered":"2024-09-18T00:57:09.796934+0000","last_unstale":"2024-09-18T01:35:02.960946+0000","last_undegraded":"2024-09-18T01:35:02.960946+0000","last_fullsized":"2024-09-18T01:35:02.960946+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T00:57:08.784496+0000","last_clean_scrub_stamp":"2024-09-18T00:57:08.784496+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T06:37:26.386956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706129,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":88996,"kb_used_data":3728,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94282844,"statfs":{"total":96636764160,"available":96545632256,"internally_reserved":0,"allocated":3817472,"data_stored":2038277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":87292357},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706128,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":115424,"kb_used_data":3788,"kb_used_omap":1,"kb_used_meta":111614,"kb_avail":94256416,"statfs":{"total":96636764160,"available":96518569984,"internally_reserved":0,"allocated":3878912,"data_stored":2628664,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":114293189},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738833,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66912,"kb_used_data":3788,"kb_used_omap":1,"kb_used_meta":63102,"kb_avail":94304928,"statfs":{"total":96636764160,"available":96568246272,"internally_reserved":0,"allocated":3878912,"data_stored":2628664,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":64616901},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:35:34.954 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-18T00:57:12.796466+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=12, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=32, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:34.954 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-18T00:57:12.796466+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=12, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=32, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:34.955 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-18T00:57:12.796466+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=12, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=32, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:34.955 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-18T00:57:12.796466+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=12, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=32, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:34.955 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-18T00:57:12.796466+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=12, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=32, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:34.955 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-18T00:57:12.796466+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=12, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=32, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:34.955 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-18T00:57:12.796466+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=12, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=32, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:34.955 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-18T00:57:12.796466+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=12, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=32, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:34.955 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-18T00:57:08.784496+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=0, tm_min=57, tm_sec=8, tm_wday=2, tm_yday=262, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=18, tm_hour=1, tm_min=35, tm_sec=32, tm_wday=2, tm_yday=262, tm_isdst=0) 2024-09-18T01:35:34.955 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-18T01:35:54.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-09-18T01:35:55.240 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-18T01:35:55.240 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-18T01:35:55.252 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1626,"stamp":"2024-09-18T01:35:54.358054+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":271320,"kb_used_data":11304,"kb_used_omap":4,"kb_used_meta":259963,"kb_avail":282844200,"statfs":{"total":289910292480,"available":289632460800,"internally_reserved":0,"allocated":11575296,"data_stored":7295605,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":4785,"internal_metadata":266202447},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002381"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":939,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:36.853736+0000","last_change":"2024-09-18T01:35:36.853766+0000","last_active":"2024-09-18T01:35:36.853736+0000","last_peered":"2024-09-18T01:35:36.853736+0000","last_clean":"2024-09-18T01:35:36.853736+0000","last_became_active":"2024-09-18T00:57:13.815654+0000","last_became_peered":"2024-09-18T00:57:13.815654+0000","last_unstale":"2024-09-18T01:35:36.853736+0000","last_undegraded":"2024-09-18T01:35:36.853736+0000","last_fullsized":"2024-09-18T01:35:36.853736+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:36.853640+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:36.853640+0000","last_clean_scrub_stamp":"2024-09-18T01:35:36.853640+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T12:17:50.309004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088783099999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":939,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:39.886578+0000","last_change":"2024-09-18T01:35:39.886601+0000","last_active":"2024-09-18T01:35:39.886578+0000","last_peered":"2024-09-18T01:35:39.886578+0000","last_clean":"2024-09-18T01:35:39.886578+0000","last_became_active":"2024-09-18T00:57:13.817123+0000","last_became_peered":"2024-09-18T00:57:13.817123+0000","last_unstale":"2024-09-18T01:35:39.886578+0000","last_undegraded":"2024-09-18T01:35:39.886578+0000","last_fullsized":"2024-09-18T01:35:39.886578+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:39.886485+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:39.886485+0000","last_clean_scrub_stamp":"2024-09-18T01:35:39.886485+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T11:42:30.350667+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.000818183,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":940,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:38.853702+0000","last_change":"2024-09-18T01:35:38.853742+0000","last_active":"2024-09-18T01:35:38.853702+0000","last_peered":"2024-09-18T01:35:38.853702+0000","last_clean":"2024-09-18T01:35:38.853702+0000","last_became_active":"2024-09-18T00:57:13.816435+0000","last_became_peered":"2024-09-18T00:57:13.816435+0000","last_unstale":"2024-09-18T01:35:38.853702+0000","last_undegraded":"2024-09-18T01:35:38.853702+0000","last_fullsized":"2024-09-18T01:35:38.853702+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:38.853593+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:38.853593+0000","last_clean_scrub_stamp":"2024-09-18T01:35:38.853593+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T12:20:58.147398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00086916700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":940,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:37.872037+0000","last_change":"2024-09-18T01:35:37.872075+0000","last_active":"2024-09-18T01:35:37.872037+0000","last_peered":"2024-09-18T01:35:37.872037+0000","last_clean":"2024-09-18T01:35:37.872037+0000","last_became_active":"2024-09-18T00:57:13.815768+0000","last_became_peered":"2024-09-18T00:57:13.815768+0000","last_unstale":"2024-09-18T01:35:37.872037+0000","last_undegraded":"2024-09-18T01:35:37.872037+0000","last_fullsized":"2024-09-18T01:35:37.872037+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:37.871938+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:37.871938+0000","last_clean_scrub_stamp":"2024-09-18T01:35:37.871938+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T13:24:29.749732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000610718,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"464'3","reported_seq":941,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:33.819412+0000","last_change":"2024-09-18T01:35:33.819456+0000","last_active":"2024-09-18T01:35:33.819412+0000","last_peered":"2024-09-18T01:35:33.819412+0000","last_clean":"2024-09-18T01:35:33.819412+0000","last_became_active":"2024-09-18T00:57:13.812378+0000","last_became_peered":"2024-09-18T00:57:13.812378+0000","last_unstale":"2024-09-18T01:35:33.819412+0000","last_undegraded":"2024-09-18T01:35:33.819412+0000","last_fullsized":"2024-09-18T01:35:33.819412+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"464'3","last_scrub_stamp":"2024-09-18T01:35:33.819305+0000","last_deep_scrub":"464'3","last_deep_scrub_stamp":"2024-09-18T01:35:33.819305+0000","last_clean_scrub_stamp":"2024-09-18T01:35:33.819305+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T10:08:58.910287+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.001073121,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":940,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:35.268270+0000","last_change":"2024-09-18T01:35:35.268290+0000","last_active":"2024-09-18T01:35:35.268270+0000","last_peered":"2024-09-18T01:35:35.268270+0000","last_clean":"2024-09-18T01:35:35.268270+0000","last_became_active":"2024-09-18T00:57:13.815977+0000","last_became_peered":"2024-09-18T00:57:13.815977+0000","last_unstale":"2024-09-18T01:35:35.268270+0000","last_undegraded":"2024-09-18T01:35:35.268270+0000","last_fullsized":"2024-09-18T01:35:35.268270+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:35.268180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:35.268180+0000","last_clean_scrub_stamp":"2024-09-18T01:35:35.268180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T04:03:53.345086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040989599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":941,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:36.273058+0000","last_change":"2024-09-18T01:35:36.273083+0000","last_active":"2024-09-18T01:35:36.273058+0000","last_peered":"2024-09-18T01:35:36.273058+0000","last_clean":"2024-09-18T01:35:36.273058+0000","last_became_active":"2024-09-18T00:57:13.816220+0000","last_became_peered":"2024-09-18T00:57:13.816220+0000","last_unstale":"2024-09-18T01:35:36.273058+0000","last_undegraded":"2024-09-18T01:35:36.273058+0000","last_fullsized":"2024-09-18T01:35:36.273058+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-18T01:35:36.272964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-18T01:35:36.272964+0000","last_clean_scrub_stamp":"2024-09-18T01:35:36.272964+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T05:37:12.705083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049982200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"464'2","reported_seq":942,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:35.828669+0000","last_change":"2024-09-18T01:35:35.828718+0000","last_active":"2024-09-18T01:35:35.828669+0000","last_peered":"2024-09-18T01:35:35.828669+0000","last_clean":"2024-09-18T01:35:35.828669+0000","last_became_active":"2024-09-18T00:57:13.815577+0000","last_became_peered":"2024-09-18T00:57:13.815577+0000","last_unstale":"2024-09-18T01:35:35.828669+0000","last_undegraded":"2024-09-18T01:35:35.828669+0000","last_fullsized":"2024-09-18T01:35:35.828669+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"464'2","last_scrub_stamp":"2024-09-18T01:35:35.828572+0000","last_deep_scrub":"464'2","last_deep_scrub_stamp":"2024-09-18T01:35:35.828572+0000","last_clean_scrub_stamp":"2024-09-18T01:35:35.828572+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T11:10:45.171482+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00066113999999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"464'99","reported_seq":1071,"reported_epoch":464,"state":"active+clean","last_fresh":"2024-09-18T01:35:34.874633+0000","last_change":"2024-09-18T01:35:34.874666+0000","last_active":"2024-09-18T01:35:34.874633+0000","last_peered":"2024-09-18T01:35:34.874633+0000","last_clean":"2024-09-18T01:35:34.874633+0000","last_became_active":"2024-09-18T00:57:09.796934+0000","last_became_peered":"2024-09-18T00:57:09.796934+0000","last_unstale":"2024-09-18T01:35:34.874633+0000","last_undegraded":"2024-09-18T01:35:34.874633+0000","last_fullsized":"2024-09-18T01:35:34.874633+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"464'99","last_scrub_stamp":"2024-09-18T01:35:34.874565+0000","last_deep_scrub":"464'99","last_deep_scrub_stamp":"2024-09-18T01:35:34.874565+0000","last_clean_scrub_stamp":"2024-09-18T01:35:34.874565+0000","objects_scrubbed":2,"log_size":99,"log_dups_size":0,"ondisk_log_size":99,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-19T09:42:27.150409+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":1180736,"data_compressed":11802,"data_compressed_allocated":73728,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":99,"ondisk_log_size":99,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654706133,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":88992,"kb_used_data":3728,"kb_used_omap":1,"kb_used_meta":85246,"kb_avail":94282848,"statfs":{"total":96636764160,"available":96545636352,"internally_reserved":0,"allocated":3817472,"data_stored":2038277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":87292357},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654706132,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":115420,"kb_used_data":3788,"kb_used_omap":1,"kb_used_meta":111614,"kb_avail":94256420,"statfs":{"total":96636764160,"available":96518574080,"internally_reserved":0,"allocated":3878912,"data_stored":2628664,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":114293189},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738837,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":66908,"kb_used_data":3788,"kb_used_omap":1,"kb_used_meta":63102,"kb_avail":94304932,"statfs":{"total":96636764160,"available":96568250368,"internally_reserved":0,"allocated":3878912,"data_stored":2628664,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":1595,"internal_metadata":64616901},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":590368,"data_compressed":5901,"data_compressed_allocated":36864,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-18T01:35:55.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2024-09-18T01:35:55.542 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-18T01:35:55.543 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-18T01:35:55.543 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2024-09-18T01:35:55.543 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:55.582 INFO:tasks.ceph.cluster1.osd.0:Stopped 2024-09-18T01:35:55.582 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2024-09-18T01:35:55.583 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:55.655 INFO:tasks.ceph.cluster1.osd.1:Stopped 2024-09-18T01:35:55.655 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2024-09-18T01:35:55.655 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:55.729 INFO:tasks.ceph.cluster1.osd.2:Stopped 2024-09-18T01:35:55.730 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-18T01:35:55.730 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2024-09-18T01:35:55.730 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:55.796 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2024-09-18T01:35:55.796 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-18T01:35:55.797 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2024-09-18T01:35:55.797 INFO:teuthology.orchestra.run:waiting for 300 2024-09-18T01:35:55.849 INFO:tasks.ceph.cluster1.mon.a:Stopped 2024-09-18T01:35:55.849 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-18T01:35:55.849 DEBUG:teuthology.orchestra.run.smithi078:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2024-09-18T01:35:55.929 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-18T01:35:55.929 DEBUG:teuthology.orchestra.run.smithi078:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2024-09-18T01:35:56.081 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-18T01:35:56.081 DEBUG:teuthology.orchestra.run.smithi078:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2024-09-18T01:35:56.170 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@smithi078.front.sepia.ceph.com 2024-09-18T01:35:56.171 DEBUG:teuthology.orchestra.run.smithi078:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2024-09-18T01:35:56.240 INFO:tasks.ceph:Archiving mon data... 2024-09-18T01:35:56.240 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686/data/cluster1.mon.a.tgz 2024-09-18T01:35:56.240 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2024-09-18T01:35:56.257 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.XWLxX8NU6d 2024-09-18T01:35:56.257 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.XWLxX8NU6d 2024-09-18T01:35:56.418 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.XWLxX8NU6d 2024-09-18T01:35:56.501 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.XWLxX8NU6d is 387KB 2024-09-18T01:35:56.569 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.XWLxX8NU6d 2024-09-18T01:35:56.584 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-18T01:35:56.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2024-09-18T01:35:56.626 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2024-09-18T01:35:56.703 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-18T01:35:56.722 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-18T01:35:56.722 INFO:tasks.ceph:Archiving crash dumps... 2024-09-18T01:35:56.724 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686/remote/smithi078/crash 2024-09-18T01:35:56.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-18T01:35:56.768 DEBUG:teuthology.misc:Transferring archived files from smithi150:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686/remote/smithi150/crash 2024-09-18T01:35:56.769 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-18T01:35:56.797 INFO:tasks.ceph:Compressing logs... 2024-09-18T01:35:56.798 DEBUG:teuthology.orchestra.run.smithi078:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-18T01:35:56.813 DEBUG:teuthology.orchestra.run.smithi150:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-18T01:35:56.857 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35680.log 2024-09-18T01:35:56.857 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35467.log 2024-09-18T01:35:56.858 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35680.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37042.log 2024-09-18T01:35:56.858 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35680.log.gz 2024-09-18T01:35:56.858 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35467.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2024-09-18T01:35:56.858 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35467.log.gz 2024-09-18T01:35:56.858 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.37042.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37168.log 2024-09-18T01:35:56.858 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37042.log.gz 2024-09-18T01:35:56.858 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37084.log 2024-09-18T01:35:56.858 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.37168.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35722.log 2024-09-18T01:35:56.858 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37168.log.gz 2024-09-18T01:35:56.858 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.37084.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2024-09-18T01:35:56.858 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37084.log.gz 2024-09-18T01:35:56.859 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35722.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35125.log 2024-09-18T01:35:56.859 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35722.log.gz 2024-09-18T01:35:56.859 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35466.log 2024-09-18T01:35:56.859 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35125.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37210.log 2024-09-18T01:35:56.859 INFO:teuthology.orchestra.run.smithi078.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.35125.log.gz 2024-09-18T01:35:56.859 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35466.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32058.log 2024-09-18T01:35:56.859 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35466.log.gz 2024-09-18T01:35:56.859 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.37210.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37210.log.gz 2024-09-18T01:35:56.859 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32018.log 2024-09-18T01:35:56.859 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.32058.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37252.log 2024-09-18T01:35:56.860 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.32058.log.gz 2024-09-18T01:35:56.860 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.32018.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.32018.log.gz 2024-09-18T01:35:56.860 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35341.log 2024-09-18T01:35:56.860 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.37252.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32098.log 2024-09-18T01:35:56.860 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37252.log.gz 2024-09-18T01:35:56.860 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35341.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35341.log.gz 2024-09-18T01:35:56.860 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35260.log 2024-09-18T01:35:56.860 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.32098.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35100.log 2024-09-18T01:35:56.860 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.32098.log.gz 2024-09-18T01:35:56.860 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35260.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35260.log.gz 2024-09-18T01:35:56.860 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31860.log 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35100.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35593.log 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35100.log.gz 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.31860.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.31860.log.gz 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34864.log 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35593.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35044.log 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35593.log.gz 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.34864.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.34864.log.gz 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37336.log 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35044.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35044.log.gz 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.37336.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37336.log.gz 2024-09-18T01:35:56.861 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2024-09-18T01:35:56.862 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32197.log 2024-09-18T01:35:56.862 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35176.log 2024-09-18T01:35:56.862 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.32197.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32265.log 2024-09-18T01:35:56.862 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.32197.log.gz 2024-09-18T01:35:56.862 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35176.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2024-09-18T01:35:56.862 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35176.log.gz 2024-09-18T01:35:56.862 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.32265.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37126.log 2024-09-18T01:35:56.862 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.32265.log.gz 2024-09-18T01:35:56.862 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36874.log 2024-09-18T01:35:56.862 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.37126.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37126.log.gz 2024-09-18T01:35:56.862 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35218.log 2024-09-18T01:35:56.863 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.36874.log: gzip 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.36874.log.gz 2024-09-18T01:35:56.863 INFO:teuthology.orchestra.run.smithi078.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.36916.log 2024-09-18T01:35:56.863 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35218.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35218.log.gz 2024-09-18T01:35:56.863 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35764.log 2024-09-18T01:35:56.863 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.36916.log: gzip -5 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.36916.log.gz 2024-09-18T01:35:56.863 INFO:teuthology.orchestra.run.smithi078.stderr: --verbose -- /var/log/ceph/ceph.tmp-client.admin.27367.log 2024-09-18T01:35:56.863 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35764.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35764.log.gz 2024-09-18T01:35:56.863 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2024-09-18T01:35:56.863 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/ceph.tmp-client.admin.27367.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36958.log 2024-09-18T01:35:56.863 INFO:teuthology.orchestra.run.smithi078.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.27367.log.gz 2024-09-18T01:35:56.863 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2024-09-18T01:35:56.864 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35342.log 2024-09-18T01:35:56.864 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-osd.0.log: /var/log/ceph/cluster1-client.admin.36958.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.36958.log.gz 2024-09-18T01:35:56.864 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35340.log 2024-09-18T01:35:56.864 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35342.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35342.log.gz 2024-09-18T01:35:56.864 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35468.log 2024-09-18T01:35:56.864 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35340.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35340.log.gz 2024-09-18T01:35:56.864 INFO:teuthology.orchestra.run.smithi078.stderr: 93.0% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2024-09-18T01:35:56.864 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35554.log 2024-09-18T01:35:56.864 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35594.log 2024-09-18T01:35:56.864 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35468.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35468.log.gz 2024-09-18T01:35:56.864 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35554.log: 0.0%gzip -- replaced with /var/log/ceph/cluster1-client.admin.35554.log.gz -5 --verbose -- /var/log/ceph/cluster1-client.admin.37294.log 2024-09-18T01:35:56.865 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-18T01:35:56.865 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.35594.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.35594.log.gz 2024-09-18T01:35:56.865 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2024-09-18T01:35:56.865 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.37294.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37294.log.gzgzip 2024-09-18T01:35:56.865 INFO:teuthology.orchestra.run.smithi078.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.37000.log 2024-09-18T01:35:56.865 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37376.log 2024-09-18T01:35:56.865 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.37000.log: /var/log/ceph/cluster1.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37000.log.gz 2024-09-18T01:35:56.865 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cluster1-client.admin.37376.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37376.log.gz 2024-09-18T01:35:56.865 INFO:teuthology.orchestra.run.smithi078.stderr: 88.1% -- replaced with /var/log/ceph/cluster1.log.gz 2024-09-18T01:35:56.892 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52817.log 2024-09-18T01:35:56.892 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71935.log 2024-09-18T01:35:56.892 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69262.log 2024-09-18T01:35:56.900 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43376.log 2024-09-18T01:35:56.901 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71935.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52817.log.gz 2024-09-18T01:35:56.901 INFO:teuthology.orchestra.run.smithi150.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71935.log.gz 2024-09-18T01:35:56.901 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.69262.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36629.log 2024-09-18T01:35:56.901 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.69262.log.gz 2024-09-18T01:35:56.901 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43376.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50507.log 2024-09-18T01:35:56.901 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.36629.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43683.log 2024-09-18T01:35:56.901 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50507.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49150.log 2024-09-18T01:35:56.901 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43683.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50507.log.gz 2024-09-18T01:35:56.901 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.60165.log 2024-09-18T01:35:56.901 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49150.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50660.log 2024-09-18T01:35:56.901 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.3.60165.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44090.log 2024-09-18T01:35:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50660.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64194.log 2024-09-18T01:35:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.44090.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50660.log.gz 2024-09-18T01:35:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65154.log 2024-09-18T01:35:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53721.log 2024-09-18T01:35:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64194.log: /var/log/ceph/cluster2-client.mirror.65154.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48089.log 2024-09-18T01:35:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53721.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65154.log.gz 2024-09-18T01:35:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46842.log 2024-09-18T01:35:56.902 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53721.log.gz 2024-09-18T01:35:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43419.log 2024-09-18T01:35:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48089.log: /var/log/ceph/cluster2-client.mirror.46842.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62451.log 2024-09-18T01:35:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43419.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61834.log 2024-09-18T01:35:56.903 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62451.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48089.log.gz 2024-09-18T01:35:56.903 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37952.log 2024-09-18T01:35:56.903 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64194.log.gz 2024-09-18T01:35:56.903 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61834.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58877.log 2024-09-18T01:35:56.903 INFO:teuthology.orchestra.run.smithi150.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62451.log.gz 2024-09-18T01:35:56.903 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61834.log.gz 2024-09-18T01:35:56.903 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37952.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50708.log 2024-09-18T01:35:56.903 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71576.log 2024-09-18T01:35:56.903 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58877.log: /var/log/ceph/cluster1-client.mirror.50708.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40801.log 2024-09-18T01:35:56.903 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50708.log.gz 2024-09-18T01:35:56.903 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71576.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37531.log 2024-09-18T01:35:56.904 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58877.log.gz 2024-09-18T01:35:56.904 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.71576.log.gz 2024-09-18T01:35:56.904 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41716.log 2024-09-18T01:35:56.904 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40801.log: /var/log/ceph/cluster1-client.admin.37531.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37531.log.gz 2024-09-18T01:35:56.904 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67622.log 2024-09-18T01:35:56.904 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.41716.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52024.log 2024-09-18T01:35:56.904 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67622.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53123.log 2024-09-18T01:35:56.904 INFO:teuthology.orchestra.run.smithi150.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67622.log.gz 2024-09-18T01:35:56.929 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70900.log 2024-09-18T01:35:56.929 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53123.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52024.log.gz 2024-09-18T01:35:56.929 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66497.log 2024-09-18T01:35:56.929 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70900.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43812.log 2024-09-18T01:35:56.929 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66497.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63716.log 2024-09-18T01:35:56.929 INFO:teuthology.orchestra.run.smithi150.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70900.log.gz 2024-09-18T01:35:56.929 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66497.log.gz 2024-09-18T01:35:56.929 INFO:teuthology.orchestra.run.smithi150.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53123.log.gz 2024-09-18T01:35:56.930 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43812.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53549.log 2024-09-18T01:35:56.930 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62196.log 2024-09-18T01:35:56.930 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63716.log: /var/log/ceph/cluster2-client.mirror.53549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40304.log 2024-09-18T01:35:56.930 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.62196.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63654.log 2024-09-18T01:35:56.930 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70524.log 2024-09-18T01:35:56.930 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63654.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45486.log 2024-09-18T01:35:56.930 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70524.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53549.log.gz 2024-09-18T01:35:56.930 INFO:teuthology.orchestra.run.smithi150.stderr:gzip 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.62196.log.gz 2024-09-18T01:35:56.930 INFO:teuthology.orchestra.run.smithi150.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39036.log 2024-09-18T01:35:56.930 INFO:teuthology.orchestra.run.smithi150.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63716.log.gz 2024-09-18T01:35:56.930 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45486.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43961.log 2024-09-18T01:35:56.931 INFO:teuthology.orchestra.run.smithi150.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70524.log.gz 2024-09-18T01:35:56.931 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54480.log 2024-09-18T01:35:56.931 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39036.log: /var/log/ceph/cluster1-client.mirror.43961.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69092.log 2024-09-18T01:35:56.931 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65733.log 2024-09-18T01:35:56.931 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63654.log.gz 2024-09-18T01:35:56.931 INFO:teuthology.orchestra.run.smithi150.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54480.log.gz 2024-09-18T01:35:56.931 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69092.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40009.log 2024-09-18T01:35:56.931 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44371.log 2024-09-18T01:35:56.931 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65733.log: /var/log/ceph/cluster1-client.mirror.40009.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66857.log 2024-09-18T01:35:56.931 INFO:teuthology.orchestra.run.smithi150.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65733.log.gz 2024-09-18T01:35:56.931 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59371.log 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44371.log: /var/log/ceph/cluster1-client.mirror.66857.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65423.log 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66857.log.gz 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59371.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62742.log 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55768.log 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65423.log: /var/log/ceph/cluster2-client.mirror.62742.log: gzip -5 --verbose -- 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65423.log.gz 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr: /var/log/ceph/cluster1-client.mirror.55926.log 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59371.log.gz 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55768.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41023.log 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55926.log: gzip -5 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55768.log.gz 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.51920.log 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55926.log.gz 2024-09-18T01:35:56.932 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41023.log: gzip 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62742.log.gz -5 2024-09-18T01:35:56.933 INFO:teuthology.orchestra.run.smithi150.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.52986.log 2024-09-18T01:35:56.933 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51920.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51650.log 2024-09-18T01:35:56.933 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52986.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46244.log 2024-09-18T01:35:56.933 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52986.log.gz 2024-09-18T01:35:56.933 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51650.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48210.log 2024-09-18T01:35:56.933 INFO:teuthology.orchestra.run.smithi150.stderr:gzip/var/log/ceph/cluster2-client.mirror.46244.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.36083.log 2024-09-18T01:35:56.933 INFO:teuthology.orchestra.run.smithi150.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51920.log.gz 2024-09-18T01:35:56.933 INFO:teuthology.orchestra.run.smithi150.stderr: 0.0%gzip -- replaced with /var/log/ceph/cluster1-client.admin.36629.log.gz 2024-09-18T01:35:56.933 INFO:teuthology.orchestra.run.smithi150.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37969.log 2024-09-18T01:35:56.933 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51650.log.gz 2024-09-18T01:35:56.933 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36083.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36083.log.gz 2024-09-18T01:35:56.934 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71347.log 2024-09-18T01:35:56.934 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37969.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56328.log 2024-09-18T01:35:56.934 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71347.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37952.log.gz 2024-09-18T01:35:56.934 INFO:teuthology.orchestra.run.smithi150.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37969.log.gz 2024-09-18T01:35:56.934 INFO:teuthology.orchestra.run.smithi150.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71347.log.gz 2024-09-18T01:35:56.934 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57657.log 2024-09-18T01:35:56.934 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56328.log: 93.4% 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.3.60165.log.gz 2024-09-18T01:35:56.934 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56328.log.gz 2024-09-18T01:35:56.934 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70424.log 2024-09-18T01:35:56.935 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57657.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.57657.log.gz 2024-09-18T01:35:56.935 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.39295.log 2024-09-18T01:35:56.935 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.70424.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40801.log.gz 2024-09-18T01:35:56.935 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70424.log.gz 2024-09-18T01:35:56.935 INFO:teuthology.orchestra.run.smithi150.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45486.log.gz 2024-09-18T01:35:56.935 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61529.log 2024-09-18T01:35:56.935 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.41023.log.gz 2024-09-18T01:35:56.935 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.0.39295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42827.log 2024-09-18T01:35:56.935 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43812.log.gz 2024-09-18T01:35:56.935 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43419.log.gz 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61529.log: /var/log/ceph/cluster2-client.mirror.48210.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40304.log.gz 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43961.log.gz 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.43683.log.gz 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.61529.log.gz 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43376.log.gz 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44371.log.gz 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 79.4% --verbose -- -- replaced with /var/log/ceph/cluster2-client.mirror.46244.log.gz 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr: /var/log/ceph/cluster1-client.mirror.59598.log 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44090.log.gz 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42827.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35755.log 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56954.log 2024-09-18T01:35:56.936 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59598.log: /var/log/ceph/cluster2-client.admin.35755.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63820.log 2024-09-18T01:35:56.937 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.35755.log.gz 2024-09-18T01:35:56.937 INFO:teuthology.orchestra.run.smithi150.stderr:gzip/var/log/ceph/cluster1-client.mirror.56954.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65485.log 2024-09-18T01:35:56.937 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63820.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47302.log 2024-09-18T01:35:56.937 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.56954.log.gz 2024-09-18T01:35:56.937 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63820.log.gz 2024-09-18T01:35:56.937 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65485.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59598.log.gz 2024-09-18T01:35:56.937 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63238.log 2024-09-18T01:35:56.937 INFO:teuthology.orchestra.run.smithi150.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.65485.log.gz 2024-09-18T01:35:56.937 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.47302.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48749.log 2024-09-18T01:35:56.937 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63238.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50732.log 2024-09-18T01:35:56.937 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48749.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62083.log 2024-09-18T01:35:56.938 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.39036.log.gz 2024-09-18T01:35:56.957 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50732.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50732.log.gz 2024-09-18T01:35:56.957 INFO:teuthology.orchestra.run.smithi150.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.40009.log.gz 2024-09-18T01:35:56.957 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62268.log 2024-09-18T01:35:56.958 INFO:teuthology.orchestra.run.smithi150.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63238.log.gz/var/log/ceph/cluster1-client.mirror.62083.log: 2024-09-18T01:35:56.958 INFO:teuthology.orchestra.run.smithi150.stderr: 95.5% 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.47302.log.gz 2024-09-18T01:35:56.958 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67307.log 2024-09-18T01:35:56.958 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.62083.log.gz 2024-09-18T01:35:56.958 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62268.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65257.log 2024-09-18T01:35:56.958 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67307.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46101.log 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67307.log.gz 2024-09-18T01:35:56.958 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:56.958 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62268.log.gz 2024-09-18T01:35:56.958 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.65257.log: -- /var/log/ceph/cluster2-client.mirror.44537.log 2024-09-18T01:35:56.958 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46101.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45895.log 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44537.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44537.log.gz 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr: 90.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70019.log 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.65257.log.gz 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45895.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.39295.log.gz 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46101.log.gz 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43789.log 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57063.log 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70019.log.gz 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43789.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.41716.log.gz 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42827.log.gz 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43789.log.gz 2024-09-18T01:35:56.959 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56906.log 2024-09-18T01:35:56.960 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57063.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39751.log 2024-09-18T01:35:56.960 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56906.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57063.log.gz 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56906.log.gz 2024-09-18T01:35:56.960 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:56.960 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41604.log 2024-09-18T01:35:56.960 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59823.log 2024-09-18T01:35:56.960 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39751.log: /var/log/ceph/cluster2-client.mirror.41604.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50957.log 2024-09-18T01:35:56.960 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59935.log 2024-09-18T01:35:56.960 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50957.log: /var/log/ceph/cluster2-client.mirror.59823.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50957.log.gz 2024-09-18T01:35:56.960 INFO:teuthology.orchestra.run.smithi150.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59823.log.gz 2024-09-18T01:35:56.960 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53808.log 2024-09-18T01:35:56.960 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.59935.log: -- /var/log/ceph/cluster2-client.mirror.40927.log 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.59935.log.gz 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53808.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55971.log 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40927.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59059.log 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55971.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53808.log.gz 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63482.log 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55971.log.gz 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.63482.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58855.log 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59059.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.63482.log.gz 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59059.log.gz 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.40927.log.gz 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32763.log 2024-09-18T01:35:56.961 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58855.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66001.log 2024-09-18T01:35:56.962 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.32763.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32763.log.gz 2024-09-18T01:35:56.962 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45772.log 2024-09-18T01:35:56.962 INFO:teuthology.orchestra.run.smithi150.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.58855.log.gz 2024-09-18T01:35:56.962 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66001.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54207.log 2024-09-18T01:35:56.962 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45772.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66001.log.gz 2024-09-18T01:35:56.962 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40121.log 2024-09-18T01:35:56.962 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54207.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45772.log.gz 2024-09-18T01:35:56.962 INFO:teuthology.orchestra.run.smithi150.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54207.log.gz 2024-09-18T01:35:56.977 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56930.log 2024-09-18T01:35:56.977 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62908.log 2024-09-18T01:35:56.977 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56930.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64947.log 2024-09-18T01:35:56.977 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.56930.log.gz 2024-09-18T01:35:56.977 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62908.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47127.log 2024-09-18T01:35:56.977 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64947.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39643.log 2024-09-18T01:35:56.978 INFO:teuthology.orchestra.run.smithi150.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.62908.log.gz 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64947.log.gz 2024-09-18T01:35:56.978 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:56.978 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47127.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47127.log.gz 2024-09-18T01:35:56.978 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64368.log 2024-09-18T01:35:56.978 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39643.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61416.log 2024-09-18T01:35:56.978 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.64368.log: -- /var/log/ceph/cluster2-client.mirror.56834.log 2024-09-18T01:35:56.978 INFO:teuthology.orchestra.run.smithi150.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64368.log.gz 2024-09-18T01:35:56.978 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61416.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53456.log 2024-09-18T01:35:56.978 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56834.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.39751.log.gz 2024-09-18T01:35:56.978 INFO:teuthology.orchestra.run.smithi150.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.39643.log.gz 2024-09-18T01:35:56.978 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56834.log.gz 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61416.log.gz 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61488.log 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43769.log 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61488.log: /var/log/ceph/cluster1-client.mirror.53456.log: 87.4% 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61488.log.gz 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.53456.log.gz 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63094.log 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63696.log 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63094.log: /var/log/ceph/cluster2-client.mirror.43769.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63094.log.gz 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40951.log 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63696.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63696.log.gz 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65175.log 2024-09-18T01:35:56.979 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40951.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42202.log 2024-09-18T01:35:56.980 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65175.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43769.log.gz 2024-09-18T01:35:56.980 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55657.log 2024-09-18T01:35:56.980 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.65175.log.gz 2024-09-18T01:35:56.980 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42202.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.40951.log.gz 2024-09-18T01:35:56.980 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44176.log 2024-09-18T01:35:56.980 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37879.log 2024-09-18T01:35:56.980 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.44176.log: /var/log/ceph/cluster1-client.mirror.55657.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55657.log.gz 2024-09-18T01:35:56.982 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66722.log 2024-09-18T01:35:56.982 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37879.log: gzip -5 --verbose 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41604.log.gz 2024-09-18T01:35:56.983 INFO:teuthology.orchestra.run.smithi150.stderr: -- /var/log/ceph/cluster2-client.mirror.53183.log 2024-09-18T01:35:56.983 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40121.log.gz 2024-09-18T01:35:56.983 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66722.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66722.log.gz 2024-09-18T01:35:56.984 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44176.log.gz 2024-09-18T01:35:56.984 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62554.log 2024-09-18T01:35:56.985 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53183.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53183.log.gz 2024-09-18T01:35:56.986 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60031.log 2024-09-18T01:35:56.988 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62554.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62554.log.gz 2024-09-18T01:35:56.989 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38654.log 2024-09-18T01:35:56.991 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60031.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60031.log.gz 2024-09-18T01:35:56.991 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56240.log 2024-09-18T01:35:56.991 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.38654.log: 57.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37879.log.gz 2024-09-18T01:35:56.991 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60721.log 2024-09-18T01:35:56.993 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56240.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.56240.log.gz 2024-09-18T01:35:56.993 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41831.log 2024-09-18T01:35:56.995 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60721.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60721.log.gz 2024-09-18T01:35:56.995 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47265.log 2024-09-18T01:35:56.996 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41831.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40141.log 2024-09-18T01:35:56.997 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47265.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41831.log.gz 2024-09-18T01:35:57.005 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53432.log 2024-09-18T01:35:57.023 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40141.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.40141.log.gz 2024-09-18T01:35:57.043 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55143.log 2024-09-18T01:35:57.044 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53432.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53432.log.gz 2024-09-18T01:35:57.053 INFO:teuthology.orchestra.run.smithi078.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2024-09-18T01:35:57.057 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71170.log 2024-09-18T01:35:57.058 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55143.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55143.log.gz 2024-09-18T01:35:57.071 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53058.log 2024-09-18T01:35:57.071 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71170.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71170.log.gz 2024-09-18T01:35:57.079 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.63482.log 2024-09-18T01:35:57.092 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53058.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.53058.log.gz 2024-09-18T01:35:57.096 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39060.log 2024-09-18T01:35:57.112 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.1.63482.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.1.63482.log.gz 2024-09-18T01:35:57.120 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66205.log 2024-09-18T01:35:57.128 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39060.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.42202.log.gz 2024-09-18T01:35:57.135 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63572.log 2024-09-18T01:35:57.135 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66205.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.66205.log.gz 2024-09-18T01:35:57.137 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.39060.log.gz 2024-09-18T01:35:57.147 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45665.log 2024-09-18T01:35:57.148 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63572.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63572.log.gz 2024-09-18T01:35:57.157 INFO:teuthology.orchestra.run.smithi078.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2024-09-18T01:35:57.168 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70822.log 2024-09-18T01:35:57.170 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.45665.log: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.45665.log.gz 2024-09-18T01:35:57.174 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48210.log.gz 2024-09-18T01:35:57.175 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62384.log 2024-09-18T01:35:57.194 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51534.log 2024-09-18T01:35:57.195 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.62384.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.62384.log.gz 2024-09-18T01:35:57.207 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67240.log 2024-09-18T01:35:57.207 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51534.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51534.log.gz 2024-09-18T01:35:57.219 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48707.log 2024-09-18T01:35:57.219 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67240.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67240.log.gz 2024-09-18T01:35:57.231 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.39554.log 2024-09-18T01:35:57.231 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48707.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48707.log.gz 2024-09-18T01:35:57.245 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48072.log 2024-09-18T01:35:57.247 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.0.39554.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67757.log 2024-09-18T01:35:57.247 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48072.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48072.log.gz 2024-09-18T01:35:57.248 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72030.log 2024-09-18T01:35:57.252 INFO:teuthology.orchestra.run.smithi150.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.0.39554.log.gz 2024-09-18T01:35:57.253 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67757.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67757.log.gz 2024-09-18T01:35:57.254 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52072.log 2024-09-18T01:35:57.256 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72030.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72030.log.gz 2024-09-18T01:35:57.257 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66069.log 2024-09-18T01:35:57.257 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52072.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52072.log.gz 2024-09-18T01:35:57.262 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72406.log 2024-09-18T01:35:57.262 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66069.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66069.log.gz 2024-09-18T01:35:57.262 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65526.log 2024-09-18T01:35:57.262 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72406.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72406.log.gz 2024-09-18T01:35:57.262 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52000.log 2024-09-18T01:35:57.262 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65526.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65526.log.gz 2024-09-18T01:35:57.273 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62155.log 2024-09-18T01:35:57.273 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52000.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52000.log.gz 2024-09-18T01:35:57.281 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53890.log 2024-09-18T01:35:57.295 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.62155.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62155.log.gz 2024-09-18T01:35:57.297 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71404.log 2024-09-18T01:35:57.298 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53890.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53890.log.gz 2024-09-18T01:35:57.306 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40483.log 2024-09-18T01:35:57.319 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71404.log: 74.8% -- replaced with /var/log/ceph/cluster1-client.mirror.71404.log.gz 2024-09-18T01:35:57.324 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71985.log 2024-09-18T01:35:57.330 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40483.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.40483.log.gz 2024-09-18T01:35:57.339 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45523.log 2024-09-18T01:35:57.341 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71985.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71985.log.gz 2024-09-18T01:35:57.341 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68607.log 2024-09-18T01:35:57.341 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45523.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45523.log.gz 2024-09-18T01:35:57.342 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48534.log 2024-09-18T01:35:57.351 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68607.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69070.log 2024-09-18T01:35:57.360 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48534.log: 87.4%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52841.log 2024-09-18T01:35:57.364 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69070.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48534.log.gz 2024-09-18T01:35:57.366 INFO:teuthology.orchestra.run.smithi150.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.68607.log.gz 2024-09-18T01:35:57.368 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65979.log 2024-09-18T01:35:57.373 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52841.log: -- replaced with /var/log/ceph/cluster2-client.mirror.69070.log.gz 2024-09-18T01:35:57.377 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46165.log 2024-09-18T01:35:57.381 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.65979.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52841.log.gz 2024-09-18T01:35:57.386 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59706.log 2024-09-18T01:35:57.390 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46165.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65979.log.gz 2024-09-18T01:35:57.400 INFO:teuthology.orchestra.run.smithi150.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46165.log.gz 2024-09-18T01:35:57.400 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36755.log 2024-09-18T01:35:57.401 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59706.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59706.log.gz 2024-09-18T01:35:57.413 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44777.log 2024-09-18T01:35:57.414 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.36755.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.36755.log.gz 2024-09-18T01:35:57.426 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43551.log 2024-09-18T01:35:57.427 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44777.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44777.log.gz 2024-09-18T01:35:57.437 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45895.log.gz 2024-09-18T01:35:57.438 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44285.log 2024-09-18T01:35:57.438 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43551.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43551.log.gz 2024-09-18T01:35:57.439 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68862.log 2024-09-18T01:35:57.446 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44285.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44285.log.gz 2024-09-18T01:35:57.446 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64174.log 2024-09-18T01:35:57.446 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.68862.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.68862.log.gz 2024-09-18T01:35:57.449 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66362.log 2024-09-18T01:35:57.449 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64174.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64174.log.gz 2024-09-18T01:35:57.449 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63362.log 2024-09-18T01:35:57.450 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66362.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66362.log.gz 2024-09-18T01:35:57.450 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60837.log 2024-09-18T01:35:57.452 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63362.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63362.log.gz 2024-09-18T01:35:57.453 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54850.log 2024-09-18T01:35:57.454 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60837.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60837.log.gz 2024-09-18T01:35:57.454 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61292.log 2024-09-18T01:35:57.455 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54850.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54850.log.gz 2024-09-18T01:35:57.456 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63218.log 2024-09-18T01:35:57.458 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61292.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61292.log.gz 2024-09-18T01:35:57.458 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45326.log 2024-09-18T01:35:57.459 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63218.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63218.log.gz 2024-09-18T01:35:57.460 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40733.log 2024-09-18T01:35:57.461 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45326.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45326.log.gz 2024-09-18T01:35:57.461 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70084.log 2024-09-18T01:35:57.462 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.40733.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59329.log 2024-09-18T01:35:57.464 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70084.log: 88.1% 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.40733.log.gz 2024-09-18T01:35:57.464 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.70084.log.gz 2024-09-18T01:35:57.464 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53275.log 2024-09-18T01:35:57.471 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59329.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59329.log.gz 2024-09-18T01:35:57.472 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45792.log 2024-09-18T01:35:57.472 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53275.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53275.log.gz 2024-09-18T01:35:57.472 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55340.log 2024-09-18T01:35:57.472 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45792.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45792.log.gz 2024-09-18T01:35:57.472 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.68644.log 2024-09-18T01:35:57.472 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55340.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55340.log.gz 2024-09-18T01:35:57.472 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37507.log 2024-09-18T01:35:57.475 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.2.68644.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.68644.log.gz 2024-09-18T01:35:57.476 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52711.log 2024-09-18T01:35:57.477 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.37507.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.37507.log.gz 2024-09-18T01:35:57.477 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58288.log 2024-09-18T01:35:57.478 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52711.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52711.log.gz 2024-09-18T01:35:57.479 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58628.log 2024-09-18T01:35:57.480 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58288.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.58288.log.gz 2024-09-18T01:35:57.480 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65113.log 2024-09-18T01:35:57.481 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58628.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58628.log.gz 2024-09-18T01:35:57.482 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56814.log 2024-09-18T01:35:57.482 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65113.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65113.log.gz 2024-09-18T01:35:57.483 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66430.log 2024-09-18T01:35:57.484 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56814.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56814.log.gz 2024-09-18T01:35:57.485 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65911.log 2024-09-18T01:35:57.485 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66430.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66430.log.gz 2024-09-18T01:35:57.486 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39667.log 2024-09-18T01:35:57.487 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.65911.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65911.log.gz 2024-09-18T01:35:57.488 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72322.log 2024-09-18T01:35:57.489 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71261.log 2024-09-18T01:35:57.489 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72322.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72322.log.gz 2024-09-18T01:35:57.490 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55878.log 2024-09-18T01:35:57.490 INFO:teuthology.orchestra.run.smithi150.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.39667.log.gz 2024-09-18T01:35:57.490 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71261.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71261.log.gz 2024-09-18T01:35:57.491 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44113.log 2024-09-18T01:35:57.492 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55878.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.55878.log.gz 2024-09-18T01:35:57.493 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57585.log 2024-09-18T01:35:57.493 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44113.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49150.log.gz 2024-09-18T01:35:57.494 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70879.log 2024-09-18T01:35:57.496 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57585.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57585.log.gz 2024-09-18T01:35:57.496 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72448.log 2024-09-18T01:35:57.496 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.70879.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.70879.log.gz 2024-09-18T01:35:57.499 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60424.log 2024-09-18T01:35:57.499 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72448.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72448.log.gz 2024-09-18T01:35:57.499 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44113.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63156.log 2024-09-18T01:35:57.499 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:57.499 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48265.log 2024-09-18T01:35:57.500 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60424.log: /var/log/ceph/cluster2-client.mirror.63156.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63156.log.gz 2024-09-18T01:35:57.500 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69516.log 2024-09-18T01:35:57.500 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.48265.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60424.log.gz 2024-09-18T01:35:57.502 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59915.log 2024-09-18T01:35:57.502 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69516.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.48265.log.gz 2024-09-18T01:35:57.502 INFO:teuthology.orchestra.run.smithi150.stderr: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69516.log.gz 2024-09-18T01:35:57.502 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62846.log 2024-09-18T01:35:57.502 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59915.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52552.log 2024-09-18T01:35:57.503 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62846.log: 89.7% 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.62846.log.gz 2024-09-18T01:35:57.503 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.59915.log.gz 2024-09-18T01:35:57.503 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45241.log 2024-09-18T01:35:57.504 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60958.log 2024-09-18T01:35:57.504 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52552.log: /var/log/ceph/cluster2-client.mirror.45241.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52552.log.gz 2024-09-18T01:35:57.507 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69426.log 2024-09-18T01:35:57.507 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60958.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45241.log.gz 2024-09-18T01:35:57.507 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60958.log.gz 2024-09-18T01:35:57.507 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36587.log 2024-09-18T01:35:57.507 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69426.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53846.log 2024-09-18T01:35:57.507 INFO:teuthology.orchestra.run.smithi150.stderr: 77.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69426.log.gz 2024-09-18T01:35:57.507 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.36587.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66880.log 2024-09-18T01:35:57.507 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53846.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.36587.log.gz 2024-09-18T01:35:57.508 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51713.log 2024-09-18T01:35:57.511 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66880.log: 89.7% 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66880.log.gz 2024-09-18T01:35:57.511 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.53846.log.gz 2024-09-18T01:35:57.511 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70722.log 2024-09-18T01:35:57.511 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.51713.log: -- /var/log/ceph/cluster2-client.mirror.69033.log 2024-09-18T01:35:57.511 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70722.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51713.log.gz 2024-09-18T01:35:57.512 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70722.log.gz 2024-09-18T01:35:57.512 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48131.log 2024-09-18T01:35:57.512 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69033.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57951.log 2024-09-18T01:35:57.512 INFO:teuthology.orchestra.run.smithi150.stderr: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69033.log.gz 2024-09-18T01:35:57.512 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48131.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70486.log 2024-09-18T01:35:57.513 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57951.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.57951.log.gz 2024-09-18T01:35:57.513 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63861.log 2024-09-18T01:35:57.515 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70486.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70486.log.gz 2024-09-18T01:35:57.516 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60118.log 2024-09-18T01:35:57.517 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63861.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63861.log.gz 2024-09-18T01:35:57.517 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54922.log 2024-09-18T01:35:57.519 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60118.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60118.log.gz 2024-09-18T01:35:57.519 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68885.log 2024-09-18T01:35:57.520 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54922.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.54922.log.gz 2024-09-18T01:35:57.520 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66182.log 2024-09-18T01:35:57.522 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.68885.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.68885.log.gz 2024-09-18T01:35:57.522 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49108.log 2024-09-18T01:35:57.523 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66182.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66182.log.gz 2024-09-18T01:35:57.523 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49891.log 2024-09-18T01:35:57.525 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49108.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49108.log.gz 2024-09-18T01:35:57.525 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57497.log 2024-09-18T01:35:57.526 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49891.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49891.log.gz 2024-09-18T01:35:57.526 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53340.log 2024-09-18T01:35:57.529 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57497.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.57497.log.gz 2024-09-18T01:35:57.529 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61993.log 2024-09-18T01:35:57.531 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53340.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53340.log.gz 2024-09-18T01:35:57.531 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45309.log 2024-09-18T01:35:57.533 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61993.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61993.log.gz 2024-09-18T01:35:57.533 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71910.log 2024-09-18T01:35:57.535 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45309.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45309.log.gz 2024-09-18T01:35:57.535 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68074.log 2024-09-18T01:35:57.535 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71910.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71910.log.gz 2024-09-18T01:35:57.536 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43597.log 2024-09-18T01:35:57.537 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.68074.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.68074.log.gz 2024-09-18T01:35:57.537 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.71404.log 2024-09-18T01:35:57.543 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43597.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43597.log.gz 2024-09-18T01:35:57.543 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42333.log 2024-09-18T01:35:57.546 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.2.71404.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.2.71404.log.gz 2024-09-18T01:35:57.546 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58336.log 2024-09-18T01:35:57.546 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42333.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61713.log 2024-09-18T01:35:57.547 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58336.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.58336.log.gz 2024-09-18T01:35:57.548 INFO:teuthology.orchestra.run.smithi150.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42333.log.gz 2024-09-18T01:35:57.548 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61858.log 2024-09-18T01:35:57.550 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61713.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61713.log.gz 2024-09-18T01:35:57.550 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53251.log 2024-09-18T01:35:57.550 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61858.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61858.log.gz 2024-09-18T01:35:57.551 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52206.log 2024-09-18T01:35:57.552 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53251.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53251.log.gz 2024-09-18T01:35:57.552 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.48593.log 2024-09-18T01:35:57.555 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52206.log: 95.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52206.log.gz 2024-09-18T01:35:57.555 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53082.log 2024-09-18T01:35:57.572 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.0.48593.log: 93.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70780.log 2024-09-18T01:35:57.573 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53082.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53082.log.gz 2024-09-18T01:35:57.573 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.0.48593.log.gz 2024-09-18T01:35:57.574 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67532.log 2024-09-18T01:35:57.575 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70780.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70780.log.gz 2024-09-18T01:35:57.576 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59509.log 2024-09-18T01:35:57.577 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67532.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67532.log.gz 2024-09-18T01:35:57.577 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48476.log 2024-09-18T01:35:57.579 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59509.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.59509.log.gz 2024-09-18T01:35:57.579 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61597.log 2024-09-18T01:35:57.579 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69092.log.gz 2024-09-18T01:35:57.579 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48476.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44242.log 2024-09-18T01:35:57.580 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61597.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61597.log.gz 2024-09-18T01:35:57.581 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55810.log 2024-09-18T01:35:57.586 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44242.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44242.log.gz 2024-09-18T01:35:57.587 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71050.log 2024-09-18T01:35:57.588 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55810.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55810.log.gz 2024-09-18T01:35:57.588 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67825.log 2024-09-18T01:35:57.588 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71050.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71050.log.gz 2024-09-18T01:35:57.590 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55075.log 2024-09-18T01:35:57.590 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67825.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67825.log.gz 2024-09-18T01:35:57.591 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43263.log 2024-09-18T01:35:57.595 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55075.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55075.log.gz 2024-09-18T01:35:57.595 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32550.log 2024-09-18T01:35:57.595 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7%/var/log/ceph/cluster1-client.mirror.43263.log: -- replaced with /var/log/ceph/cluster2-client.mirror.48749.log.gz 2024-09-18T01:35:57.595 INFO:teuthology.orchestra.run.smithi150.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.43263.log.gz 2024-09-18T01:35:57.595 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.37178.log 2024-09-18T01:35:57.596 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59349.log 2024-09-18T01:35:57.596 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.0.37178.log: /var/log/ceph/cluster2-client.admin.32550.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32550.log.gz 2024-09-18T01:35:57.596 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56469.log 2024-09-18T01:35:57.596 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65402.log 2024-09-18T01:35:57.598 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56469.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59349.log.gz 2024-09-18T01:35:57.598 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49071.log 2024-09-18T01:35:57.598 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65402.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56469.log.gz 2024-09-18T01:35:57.598 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41516.log 2024-09-18T01:35:57.598 INFO:teuthology.orchestra.run.smithi150.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65402.log.gz 2024-09-18T01:35:57.598 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41516.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41516.log.gz 2024-09-18T01:35:57.599 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46842.log.gz 2024-09-18T01:35:57.636 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54629.log 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63011.log 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54629.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56376.log 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.54629.log.gz 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63011.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45224.log 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56376.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63011.log.gz 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55633.log 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.56376.log.gz 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.45224.log: --verbose -- /var/log/ceph/cluster1-client.mirror.50684.log 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49071.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45224.log.gz 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69553.log 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55633.log: /var/log/ceph/cluster1-client.mirror.50684.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58264.log 2024-09-18T01:35:57.637 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.55633.log.gz 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50684.log.gz/var/log/ceph/cluster2-client.mirror.69553.log: 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70856.log 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69553.log.gz 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58264.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65278.log 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7%/var/log/ceph/cluster1-client.mirror.70856.log: -- replaced with /var/log/ceph/cluster1-client.mirror.58264.log.gz 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45923.log 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.70856.log.gz 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65278.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51448.log 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47265.log.gz 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45923.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65278.log.gz 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69763.log 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45923.log.gz 2024-09-18T01:35:57.638 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44497.log 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51448.log: /var/log/ceph/cluster2-client.mirror.69763.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65464.log 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62107.log 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr: 88.5%/var/log/ceph/cluster2-client.mirror.65464.log: -- replaced with /var/log/ceph/cluster2-client.mirror.44497.log.gz 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56424.log 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.62107.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.65464.log.gz 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53629.log 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.62107.log.gz 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69763.log.gz 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56424.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63300.log 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56424.log.gz/var/log/ceph/cluster2-client.mirror.53629.log: 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr:gzip 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51448.log.gz 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66317.log 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63300.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55523.log 2024-09-18T01:35:57.639 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53629.log.gz 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66317.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64063.log 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66317.log.gz 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55523.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61030.log 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64063.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42807.log 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55523.log.gz 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63300.log.gz/var/log/ceph/cluster1-client.mirror.61030.log: 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49363.log 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42807.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65866.log 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61030.log.gz 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49363.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71092.log 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49363.log.gz 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.65866.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49874.log 2024-09-18T01:35:57.640 INFO:teuthology.orchestra.run.smithi150.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65866.log.gz 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71092.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65753.log 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64063.log.gz 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49874.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49874.log.gz 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71092.log.gz 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.64063.log 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65753.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66137.log 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.42807.log.gz 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.0.64063.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59308.log 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66137.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67172.log 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9%/var/log/ceph/cluster1-client.mirror.59308.log: -- replaced with /var/log/ceph/cluster1-client.mirror.66137.log.gz 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65051.log 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.59308.log.gz 2024-09-18T01:35:57.641 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65753.log.gz 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67172.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67172.log.gz 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48690.log 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65051.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60653.log 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48690.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62039.log 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.65051.log.gz 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48690.log.gz 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.60653.log: -- /var/log/ceph/cluster2-client.mirror.55481.log 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62039.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54543.log 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55481.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62039.log.gz 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37777.log 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60653.log.gz 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54543.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56858.log 2024-09-18T01:35:57.642 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.37777.log: 92.5% 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54543.log.gz 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.55481.log.gz 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65195.log 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56858.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60675.log 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.56858.log.gz 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65195.log: /var/log/ceph/cluster2-client.mirror.60675.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60675.log.gz 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65195.log.gz 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42988.log 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.0.37178.log.gz 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr: -- /var/log/ceph/cluster2-client.mirror.45261.log 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42988.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48933.log 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45261.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49812.log 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45261.log.gz/var/log/ceph/cluster2-client.mirror.48933.log: 2024-09-18T01:35:57.643 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48933.log.gz 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.67667.log 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49812.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60789.log 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67667.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40639.log 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67667.log.gz 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49812.log.gz 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr: 80.0% -- replaced with /var/log/ceph/cluster2-client.admin.37777.log.gz 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35995.log 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60789.log: /var/log/ceph/cluster1-client.mirror.40639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55033.log 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60789.log.gz 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.35995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65381.log 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35995.log.gz 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58240.log 2024-09-18T01:35:57.644 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67487.log 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65381.log: 89.8%/var/log/ceph/cluster2-client.mirror.58240.log: -- replaced with /var/log/ceph/cluster2-client.mirror.55033.log.gz 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58240.log.gz 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44219.log 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.64063.log.gz 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67487.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55278.log 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9%/var/log/ceph/cluster1-client.mirror.44219.log: gzip -- replaced with /var/log/ceph/cluster1-client.mirror.67487.log.gz 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59417.log 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55278.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65381.log.gz 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55278.log.gz 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.37259.log 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49694.log 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59417.log: /var/log/ceph/cluster2-client.mirror.2.37259.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59417.log.gz 2024-09-18T01:35:57.645 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51602.log 2024-09-18T01:35:57.646 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52440.log 2024-09-18T01:35:57.646 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51602.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37259.log 2024-09-18T01:35:57.646 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51602.log.gz 2024-09-18T01:35:57.646 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44219.log.gz 2024-09-18T01:35:57.660 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52440.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52440.log.gz 2024-09-18T01:35:57.660 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46885.log 2024-09-18T01:35:57.660 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.37259.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47428.log 2024-09-18T01:35:57.660 INFO:teuthology.orchestra.run.smithi150.stderr:gzip/var/log/ceph/cluster2-client.mirror.46885.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61882.log 2024-09-18T01:35:57.660 INFO:teuthology.orchestra.run.smithi150.stderr: 79.6%/var/log/ceph/cluster2-client.mirror.47428.log: -- replaced with /var/log/ceph/cluster2-client.mirror.46885.log.gz 2024-09-18T01:35:57.660 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40267.log 2024-09-18T01:35:57.660 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose 87.5%/var/log/ceph/cluster1-client.mirror.61882.log: -- /var/log/ceph/cluster2-client.mirror.62866.log 2024-09-18T01:35:57.660 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.47428.log.gz 2024-09-18T01:35:57.660 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61882.log.gz 2024-09-18T01:35:57.660 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40267.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45853.log 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62866.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70566.log 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45853.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45853.log.gz 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62866.log.gz 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40661.log 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70566.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37259.log.gz 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63944.log 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40661.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46261.log 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60564.log 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46261.log: /var/log/ceph/cluster2-client.mirror.63944.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63944.log.gz 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46261.log.gz 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40975.log 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51980.log 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40975.log: /var/log/ceph/cluster1-client.mirror.60564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64657.log 2024-09-18T01:35:57.661 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51980.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60564.log.gz 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61095.log 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51980.log.gz/var/log/ceph/cluster2-client.mirror.64657.log: 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64657.log.gz 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61372.log 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40661.log.gz 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61095.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71494.log 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61372.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61095.log.gz 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67465.log 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71494.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61372.log.gz 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71494.log.gz 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.40639.log.gz 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.62218.log 2024-09-18T01:35:57.662 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.40975.log.gz 2024-09-18T01:35:57.663 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67465.log: 86.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71133.log 2024-09-18T01:35:57.663 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.67465.log.gz 2024-09-18T01:35:57.663 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62218.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51094.log 2024-09-18T01:35:57.663 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71133.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62218.log.gz 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71133.log.gz 2024-09-18T01:35:57.663 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:57.663 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58586.log 2024-09-18T01:35:57.663 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.51094.log: -- /var/log/ceph/cluster2-client.mirror.69369.log 2024-09-18T01:35:57.663 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58586.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58586.log.gz 2024-09-18T01:35:57.673 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66227.log 2024-09-18T01:35:57.673 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69369.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69369.log.gz 2024-09-18T01:35:57.673 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47745.log 2024-09-18T01:35:57.673 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66227.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66227.log.gz 2024-09-18T01:35:57.673 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44637.log 2024-09-18T01:35:57.673 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47745.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47745.log.gz 2024-09-18T01:35:57.673 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35699.log 2024-09-18T01:35:57.674 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44637.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44637.log.gz 2024-09-18T01:35:57.674 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71305.log 2024-09-18T01:35:57.674 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.35699.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35699.log.gz 2024-09-18T01:35:57.674 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58963.log 2024-09-18T01:35:57.674 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71305.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71305.log.gz 2024-09-18T01:35:57.674 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55950.log 2024-09-18T01:35:57.674 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58963.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58963.log.gz 2024-09-18T01:35:57.674 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.52206.log 2024-09-18T01:35:57.674 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55950.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55950.log.gz 2024-09-18T01:35:57.674 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41226.log 2024-09-18T01:35:57.676 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.0.52206.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.42988.log.gz 2024-09-18T01:35:57.681 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68457.log 2024-09-18T01:35:57.683 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.41226.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51094.log.gz 2024-09-18T01:35:57.686 INFO:teuthology.orchestra.run.smithi150.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40267.log.gz 2024-09-18T01:35:57.689 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69613.log 2024-09-18T01:35:57.690 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.68457.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.68457.log.gz 2024-09-18T01:35:57.693 INFO:teuthology.orchestra.run.smithi150.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.41226.log.gz 2024-09-18T01:35:57.706 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51186.log 2024-09-18T01:35:57.707 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.69613.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.69613.log.gz 2024-09-18T01:35:57.714 INFO:teuthology.orchestra.run.smithi150.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.52206.log.gz 2024-09-18T01:35:57.715 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70822.log.gz 2024-09-18T01:35:57.716 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58742.log 2024-09-18T01:35:57.717 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51186.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51186.log.gz 2024-09-18T01:35:57.717 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53866.log 2024-09-18T01:35:57.718 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58742.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.58742.log.gz 2024-09-18T01:35:57.718 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65844.log 2024-09-18T01:35:57.720 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53866.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53866.log.gz 2024-09-18T01:35:57.720 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56653.log 2024-09-18T01:35:57.721 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.65844.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65844.log.gz 2024-09-18T01:35:57.721 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70615.log 2024-09-18T01:35:57.721 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56653.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.56653.log.gz 2024-09-18T01:35:57.722 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63799.log 2024-09-18T01:35:57.724 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.70615.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.70615.log.gz 2024-09-18T01:35:57.724 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71192.log 2024-09-18T01:35:57.725 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63799.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63799.log.gz 2024-09-18T01:35:57.725 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39875.log 2024-09-18T01:35:57.735 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71192.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44577.log 2024-09-18T01:35:57.741 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39875.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66767.log 2024-09-18T01:35:57.742 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44577.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39875.log.gz 2024-09-18T01:35:57.755 INFO:teuthology.orchestra.run.smithi150.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44577.log.gz 2024-09-18T01:35:57.756 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47725.log 2024-09-18T01:35:57.756 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66767.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66767.log.gz 2024-09-18T01:35:57.777 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.2.37259.log.gz 2024-09-18T01:35:57.777 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42474.log 2024-09-18T01:35:57.787 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47725.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66452.log 2024-09-18T01:35:57.792 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42474.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61951.log 2024-09-18T01:35:57.792 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66452.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66452.log.gz 2024-09-18T01:35:57.792 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46303.log 2024-09-18T01:35:57.795 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61951.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61951.log.gz 2024-09-18T01:35:57.795 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46635.log 2024-09-18T01:35:57.805 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46303.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42474.log.gz 2024-09-18T01:35:57.810 INFO:teuthology.orchestra.run.smithi078.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2024-09-18T01:35:57.814 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45584.log 2024-09-18T01:35:57.815 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46635.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46635.log.gz 2024-09-18T01:35:57.833 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66295.log 2024-09-18T01:35:57.834 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45584.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45584.log.gz 2024-09-18T01:35:57.842 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71885.log 2024-09-18T01:35:57.842 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66295.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66295.log.gz 2024-09-18T01:35:57.857 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48831.log 2024-09-18T01:35:57.858 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71885.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71885.log.gz 2024-09-18T01:35:57.863 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39917.log 2024-09-18T01:35:57.871 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48831.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68050.log 2024-09-18T01:35:57.881 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39917.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48131.log.gz 2024-09-18T01:35:57.884 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39917.log.gz 2024-09-18T01:35:57.887 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57224.log 2024-09-18T01:35:57.888 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.68050.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68050.log.gz 2024-09-18T01:35:57.907 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56629.log 2024-09-18T01:35:57.909 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57224.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57224.log.gz 2024-09-18T01:35:57.913 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49071.log.gz 2024-09-18T01:35:57.929 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38654.log.gz 2024-09-18T01:35:57.932 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.71763.log 2024-09-18T01:35:57.933 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56629.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49954.log 2024-09-18T01:35:57.933 INFO:teuthology.orchestra.run.smithi150.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.56629.log.gz 2024-09-18T01:35:57.935 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.0.71763.log: 91.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.71763.log.gz 2024-09-18T01:35:57.938 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70680.log 2024-09-18T01:35:57.938 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49954.log: 79.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49954.log.gz 2024-09-18T01:35:57.938 INFO:teuthology.orchestra.run.smithi150.stderr:gzip 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48476.log.gz 2024-09-18T01:35:57.938 INFO:teuthology.orchestra.run.smithi150.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37606.log 2024-09-18T01:35:57.939 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48910.log 2024-09-18T01:35:57.939 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.70680.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70680.log.gz/var/log/ceph/cluster1-client.mirror.37606.log: 2024-09-18T01:35:57.939 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68992.log 2024-09-18T01:35:57.939 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48910.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55501.log 2024-09-18T01:35:57.939 INFO:teuthology.orchestra.run.smithi150.stderr: 53.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37606.log.gz 2024-09-18T01:35:57.940 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68992.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.68992.log.gz 2024-09-18T01:35:57.940 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51942.log 2024-09-18T01:35:57.942 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55501.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55501.log.gz 2024-09-18T01:35:57.942 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55902.log 2024-09-18T01:35:57.944 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51942.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51942.log.gz 2024-09-18T01:35:57.944 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46084.log 2024-09-18T01:35:57.945 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55902.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55902.log.gz 2024-09-18T01:35:57.945 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46519.log 2024-09-18T01:35:57.946 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46084.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46084.log.gz 2024-09-18T01:35:57.946 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53986.log 2024-09-18T01:35:57.947 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46519.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46519.log.gz 2024-09-18T01:35:57.947 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55388.log 2024-09-18T01:35:57.948 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53986.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53986.log.gz 2024-09-18T01:35:57.949 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46440.log 2024-09-18T01:35:57.950 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55388.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55388.log.gz 2024-09-18T01:35:57.950 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71511.log 2024-09-18T01:35:57.951 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46440.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46440.log.gz 2024-09-18T01:35:57.951 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64256.log 2024-09-18T01:35:57.953 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71511.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71511.log.gz 2024-09-18T01:35:57.953 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42633.log 2024-09-18T01:35:57.956 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64256.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64256.log.gz 2024-09-18T01:35:57.956 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37913.log 2024-09-18T01:35:57.957 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.42633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65671.log 2024-09-18T01:35:57.958 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37913.log: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37913.log.gz 2024-09-18T01:35:57.960 INFO:teuthology.orchestra.run.smithi150.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.42633.log.gz 2024-09-18T01:35:57.962 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71192.log.gz 2024-09-18T01:35:57.969 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66115.log 2024-09-18T01:35:57.970 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65671.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71860.log 2024-09-18T01:35:57.970 INFO:teuthology.orchestra.run.smithi150.stderr: 88.4%/var/log/ceph/cluster1-client.mirror.66115.log: -- replaced with /var/log/ceph/cluster2-client.mirror.65671.log.gz 2024-09-18T01:35:57.970 INFO:teuthology.orchestra.run.smithi150.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66115.log.gz 2024-09-18T01:35:57.970 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51578.log 2024-09-18T01:35:57.970 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71860.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71860.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66745.log 2024-09-18T01:35:57.970 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:57.970 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53145.log 2024-09-18T01:35:57.970 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66745.log: /var/log/ceph/cluster1-client.mirror.51578.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66745.log.gz 2024-09-18T01:35:57.970 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51578.log.gz 2024-09-18T01:35:57.970 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.45665.log 2024-09-18T01:35:57.970 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41581.log 2024-09-18T01:35:57.972 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53145.log: /var/log/ceph/cluster2-client.mirror.0.45665.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53145.log.gz 2024-09-18T01:35:57.972 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51554.log 2024-09-18T01:35:57.972 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41581.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55460.log 2024-09-18T01:35:57.977 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41581.log.gz 2024-09-18T01:35:57.978 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51554.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51554.log.gz 2024-09-18T01:35:57.979 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57477.log 2024-09-18T01:35:57.980 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55460.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55460.log.gz 2024-09-18T01:35:57.980 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42037.log 2024-09-18T01:35:57.982 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57477.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57477.log.gz 2024-09-18T01:35:57.982 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72698.log 2024-09-18T01:35:57.984 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42037.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43617.log 2024-09-18T01:35:57.984 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72698.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72698.log.gz 2024-09-18T01:35:57.984 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70245.log 2024-09-18T01:35:57.987 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43617.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.45665.log.gz 2024-09-18T01:35:57.988 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44517.log 2024-09-18T01:35:57.988 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70245.log: 85.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70245.log.gz 2024-09-18T01:35:57.989 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41903.log 2024-09-18T01:35:57.989 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44517.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44517.log.gz 2024-09-18T01:35:57.990 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53914.log 2024-09-18T01:35:57.992 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41903.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66272.log 2024-09-18T01:35:57.992 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53914.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.41903.log.gz 2024-09-18T01:35:57.992 INFO:teuthology.orchestra.run.smithi150.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53914.log.gz 2024-09-18T01:35:57.992 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57316.log 2024-09-18T01:35:57.994 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66272.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66272.log.gz 2024-09-18T01:35:57.994 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69743.log 2024-09-18T01:35:57.994 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57316.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42037.log.gz 2024-09-18T01:35:57.994 INFO:teuthology.orchestra.run.smithi150.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57316.log.gz 2024-09-18T01:35:57.995 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67735.log 2024-09-18T01:35:57.995 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43617.log.gz 2024-09-18T01:35:57.997 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69743.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49029.log 2024-09-18T01:35:57.997 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67735.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.69743.log.gz 2024-09-18T01:35:57.997 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67735.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55236.log 2024-09-18T01:35:57.997 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:57.997 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38770.log 2024-09-18T01:35:57.998 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55236.log: /var/log/ceph/cluster2-client.mirror.49029.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49029.log.gz 2024-09-18T01:35:57.998 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46457.log 2024-09-18T01:35:57.999 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.38770.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55236.log.gz 2024-09-18T01:35:57.999 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68364.log 2024-09-18T01:35:57.999 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46457.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.38770.log.gz 2024-09-18T01:35:57.999 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.69800.log 2024-09-18T01:35:58.000 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.68364.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68364.log.gz 2024-09-18T01:35:58.000 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46457.log.gz 2024-09-18T01:35:58.001 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56581.log 2024-09-18T01:35:58.001 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37845.log 2024-09-18T01:35:58.001 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.0.69800.log: /var/log/ceph/cluster2-client.mirror.56581.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56581.log.gz 2024-09-18T01:35:58.002 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62659.log 2024-09-18T01:35:58.003 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37845.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.48265.log 2024-09-18T01:35:58.003 INFO:teuthology.orchestra.run.smithi150.stderr: 2.3% -- replaced with /var/log/ceph/cluster2-client.mirror.37845.log.gz 2024-09-18T01:35:58.005 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62659.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62659.log.gz 2024-09-18T01:35:58.005 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54970.log 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.0.69800.log.gz 2024-09-18T01:35:58.005 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:58.005 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.0.48265.log: --verbose -- /var/log/ceph/cluster2-client.admin.32683.log 2024-09-18T01:35:58.007 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54970.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54970.log.gz 2024-09-18T01:35:58.007 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67330.log 2024-09-18T01:35:58.007 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.32683.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32683.log.gz 2024-09-18T01:35:58.009 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48168.log 2024-09-18T01:35:58.009 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67330.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67330.log.gz 2024-09-18T01:35:58.009 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53412.log 2024-09-18T01:35:58.010 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48168.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48168.log.gz 2024-09-18T01:35:58.010 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52419.log 2024-09-18T01:35:58.012 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53412.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53412.log.gz 2024-09-18T01:35:58.012 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54521.log 2024-09-18T01:35:58.013 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52419.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52419.log.gz 2024-09-18T01:35:58.014 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72196.log 2024-09-18T01:35:58.016 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54521.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.54521.log.gz 2024-09-18T01:35:58.016 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43215.log 2024-09-18T01:35:58.017 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72196.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72196.log.gz 2024-09-18T01:35:58.017 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72616.log 2024-09-18T01:35:58.018 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43215.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43215.log.gz 2024-09-18T01:35:58.019 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43127.log 2024-09-18T01:35:58.020 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72616.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72616.log.gz 2024-09-18T01:35:58.020 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48109.log 2024-09-18T01:35:58.020 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.43127.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.43127.log.gz 2024-09-18T01:35:58.021 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43053.log 2024-09-18T01:35:58.022 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48109.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48109.log.gz 2024-09-18T01:35:58.022 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41444.log 2024-09-18T01:35:58.023 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43053.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54408.log 2024-09-18T01:35:58.024 INFO:teuthology.orchestra.run.smithi150.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43053.log.gz 2024-09-18T01:35:58.026 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41444.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39147.log 2024-09-18T01:35:58.026 INFO:teuthology.orchestra.run.smithi150.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41444.log.gz 2024-09-18T01:35:58.026 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54408.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54408.log.gz 2024-09-18T01:35:58.027 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43726.log 2024-09-18T01:35:58.027 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39147.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41420.log 2024-09-18T01:35:58.028 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43726.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.48265.log.gz 2024-09-18T01:35:58.030 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.68364.log 2024-09-18T01:35:58.030 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41420.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45027.log 2024-09-18T01:35:58.033 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.2.68364.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43726.log.gz 2024-09-18T01:35:58.036 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39731.log 2024-09-18T01:35:58.036 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45027.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45027.log.gz 2024-09-18T01:35:58.036 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47920.log 2024-09-18T01:35:58.036 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.2.68364.log.gz 2024-09-18T01:35:58.036 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39731.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39961.log 2024-09-18T01:35:58.036 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47920.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66385.log 2024-09-18T01:35:58.039 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39961.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47920.log.gz 2024-09-18T01:35:58.039 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61312.log 2024-09-18T01:35:58.039 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65092.log 2024-09-18T01:35:58.039 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61312.log: /var/log/ceph/cluster1-client.mirror.66385.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66385.log.gz 2024-09-18T01:35:58.039 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39147.log.gz 2024-09-18T01:35:58.040 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47804.log 2024-09-18T01:35:58.040 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65092.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41420.log.gz 2024-09-18T01:35:58.040 INFO:teuthology.orchestra.run.smithi150.stderr: 88.0% 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.39961.log.gz 2024-09-18T01:35:58.040 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.65092.log.gz 2024-09-18T01:35:58.040 INFO:teuthology.orchestra.run.smithi150.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.61312.log.gz 2024-09-18T01:35:58.040 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.47302.log 2024-09-18T01:35:58.040 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44757.log 2024-09-18T01:35:58.048 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47804.log: /var/log/ceph/cluster2-client.mirror.3.47302.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.39731.log.gz 2024-09-18T01:35:58.048 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65779.log 2024-09-18T01:35:58.063 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44757.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44757.log.gz 2024-09-18T01:35:58.063 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39295.log 2024-09-18T01:35:58.063 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.65779.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.65779.log.gz 2024-09-18T01:35:58.070 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44900.log 2024-09-18T01:35:58.079 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50037.log 2024-09-18T01:35:58.079 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.44900.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44900.log.gz 2024-09-18T01:35:58.082 INFO:teuthology.orchestra.run.smithi150.stderr: 82.2% -- replaced with /var/log/ceph/cluster2-client.mirror.39295.log.gz 2024-09-18T01:35:58.096 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53227.log 2024-09-18T01:35:58.096 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50037.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.50037.log.gz 2024-09-18T01:35:58.099 INFO:teuthology.orchestra.run.smithi150.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.3.47302.log.gz 2024-09-18T01:35:58.099 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48052.log 2024-09-18T01:35:58.100 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53227.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.53227.log.gz 2024-09-18T01:35:58.100 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49215.log 2024-09-18T01:35:58.114 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48052.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57565.log 2024-09-18T01:35:58.114 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49215.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49215.log.gz 2024-09-18T01:35:58.133 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60335.log 2024-09-18T01:35:58.133 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57565.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57565.log.gz 2024-09-18T01:35:58.140 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48831.log.gz 2024-09-18T01:35:58.142 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55095.log 2024-09-18T01:35:58.143 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60335.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60335.log.gz 2024-09-18T01:35:58.143 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60383.log 2024-09-18T01:35:58.144 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55095.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55095.log.gz 2024-09-18T01:35:58.144 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37828.log 2024-09-18T01:35:58.145 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60383.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.60383.log.gz 2024-09-18T01:35:58.145 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35780.log 2024-09-18T01:35:58.147 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37828.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67217.log 2024-09-18T01:35:58.147 INFO:teuthology.orchestra.run.smithi150.stderr: 54.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37828.log.gz 2024-09-18T01:35:58.149 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.35780.log: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.35780.log.gz 2024-09-18T01:35:58.149 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46004.log 2024-09-18T01:35:58.149 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67217.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67217.log.gz 2024-09-18T01:35:58.149 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67352.log 2024-09-18T01:35:58.151 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46004.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46004.log.gz 2024-09-18T01:35:58.152 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51331.log 2024-09-18T01:35:58.152 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67352.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67352.log.gz 2024-09-18T01:35:58.152 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41068.log 2024-09-18T01:35:58.154 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51331.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51331.log.gz 2024-09-18T01:35:58.154 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58312.log 2024-09-18T01:35:58.154 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41068.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64409.log 2024-09-18T01:35:58.155 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58312.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58312.log.gz 2024-09-18T01:35:58.155 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52906.log 2024-09-18T01:35:58.157 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64409.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64409.log.gz 2024-09-18T01:35:58.157 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66925.log 2024-09-18T01:35:58.162 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52906.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41068.log.gz 2024-09-18T01:35:58.164 INFO:teuthology.orchestra.run.smithi150.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52906.log.gz 2024-09-18T01:35:58.165 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36713.log 2024-09-18T01:35:58.167 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66925.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66925.log.gz 2024-09-18T01:35:58.167 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57886.log 2024-09-18T01:35:58.167 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.36713.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.36713.log.gz 2024-09-18T01:35:58.167 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42537.log 2024-09-18T01:35:58.169 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57886.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57886.log.gz 2024-09-18T01:35:58.169 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43527.log 2024-09-18T01:35:58.169 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42537.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70147.log 2024-09-18T01:35:58.170 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43527.log: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.43527.log.gz 2024-09-18T01:35:58.171 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59802.log 2024-09-18T01:35:58.172 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70147.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70147.log.gz 2024-09-18T01:35:58.172 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66947.log 2024-09-18T01:35:58.176 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59802.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59802.log.gz 2024-09-18T01:35:58.176 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66790.log 2024-09-18T01:35:58.176 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66947.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66947.log.gz 2024-09-18T01:35:58.176 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65505.log 2024-09-18T01:35:58.176 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42537.log.gz 2024-09-18T01:35:58.176 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66790.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66790.log.gz 2024-09-18T01:35:58.176 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46922.log 2024-09-18T01:35:58.179 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65505.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65505.log.gz 2024-09-18T01:35:58.179 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56175.log 2024-09-18T01:35:58.180 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46922.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46922.log.gz 2024-09-18T01:35:58.180 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46340.log 2024-09-18T01:35:58.182 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56175.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56175.log.gz 2024-09-18T01:35:58.182 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63424.log 2024-09-18T01:35:58.182 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46340.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46340.log.gz 2024-09-18T01:35:58.183 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42164.log 2024-09-18T01:35:58.186 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63424.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63424.log.gz 2024-09-18T01:35:58.186 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47223.log 2024-09-18T01:35:58.186 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42164.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65319.log 2024-09-18T01:35:58.188 INFO:teuthology.orchestra.run.smithi150.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42164.log.gz 2024-09-18T01:35:58.190 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47223.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47223.log.gz 2024-09-18T01:35:58.190 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55991.log 2024-09-18T01:35:58.193 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65319.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65319.log.gz 2024-09-18T01:35:58.193 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66340.log 2024-09-18T01:35:58.195 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55991.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55991.log.gz 2024-09-18T01:35:58.195 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63342.log 2024-09-18T01:35:58.197 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66340.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66340.log.gz 2024-09-18T01:35:58.197 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70105.log 2024-09-18T01:35:58.198 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63342.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63342.log.gz 2024-09-18T01:35:58.198 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38944.log 2024-09-18T01:35:58.199 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70105.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70105.log.gz 2024-09-18T01:35:58.199 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71282.log 2024-09-18T01:35:58.201 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.38944.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41111.log 2024-09-18T01:35:58.201 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38944.log.gz 2024-09-18T01:35:58.203 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71282.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71282.log.gz 2024-09-18T01:35:58.203 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47725.log.gz 2024-09-18T01:35:58.203 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37221.log 2024-09-18T01:35:58.203 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43311.log 2024-09-18T01:35:58.203 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41111.log: /var/log/ceph/cluster1-client.mirror.37221.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60496.log 2024-09-18T01:35:58.210 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55830.log 2024-09-18T01:35:58.210 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60496.log: /var/log/ceph/cluster1-client.mirror.43311.log: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43311.log.gz 2024-09-18T01:35:58.210 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42517.log 2024-09-18T01:35:58.210 INFO:teuthology.orchestra.run.smithi150.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60496.log.gz 2024-09-18T01:35:58.210 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55830.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61223.log 2024-09-18T01:35:58.210 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55830.log.gz 2024-09-18T01:35:58.210 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42517.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60813.log 2024-09-18T01:35:58.211 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61223.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44414.log 2024-09-18T01:35:58.211 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61223.log.gz 2024-09-18T01:35:58.211 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60813.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60404.log 2024-09-18T01:35:58.211 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44414.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60813.log.gz 2024-09-18T01:35:58.211 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.41331.log 2024-09-18T01:35:58.211 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60404.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60404.log.gz 2024-09-18T01:35:58.226 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56352.log 2024-09-18T01:35:58.226 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.2.41331.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53673.log 2024-09-18T01:35:58.226 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56352.log: 93.4% 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56352.log.gz 2024-09-18T01:35:58.226 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.44414.log.gz 2024-09-18T01:35:58.226 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40057.log 2024-09-18T01:35:58.226 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.53673.log: /var/log/ceph/cluster2-osd.2.log 2024-09-18T01:35:58.226 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.53673.log.gz 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40057.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70636.log 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-osd.2.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.40057.log.gz 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63445.log 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.70636.log: --verbose -- /var/log/ceph/cluster1-client.mirror.1.37059.log 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63445.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70636.log.gz 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71071.log 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63445.log.gz 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.1.37059.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67150.log 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57633.log 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67150.log: /var/log/ceph/cluster2-client.mirror.71071.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71071.log.gz 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67150.log.gz 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51231.log 2024-09-18T01:35:58.227 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49911.log 2024-09-18T01:35:58.228 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57633.log: /var/log/ceph/cluster2-client.mirror.51231.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41111.log.gz 2024-09-18T01:35:58.228 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.57633.log.gz 2024-09-18T01:35:58.228 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67960.log 2024-09-18T01:35:58.228 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49911.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49911.log.gz 2024-09-18T01:35:58.228 INFO:teuthology.orchestra.run.smithi150.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51231.log.gz 2024-09-18T01:35:58.228 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63613.log 2024-09-18T01:35:58.228 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67960.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67960.log.gz 2024-09-18T01:35:58.228 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40555.log 2024-09-18T01:35:58.231 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63613.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63613.log.gz 2024-09-18T01:35:58.231 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51002.log 2024-09-18T01:35:58.231 INFO:teuthology.orchestra.run.smithi150.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.37221.log.gz 2024-09-18T01:35:58.231 INFO:teuthology.orchestra.run.smithi150.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.40555.log.gz 2024-09-18T01:35:58.231 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67442.log 2024-09-18T01:35:58.231 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51210.log 2024-09-18T01:35:58.232 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51002.log: /var/log/ceph/cluster1-client.mirror.67442.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.67442.log.gz 2024-09-18T01:35:58.232 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44434.log 2024-09-18T01:35:58.232 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51210.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51002.log.gz 2024-09-18T01:35:58.232 INFO:teuthology.orchestra.run.smithi150.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51210.log.gz 2024-09-18T01:35:58.232 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67285.log 2024-09-18T01:35:58.234 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56605.log 2024-09-18T01:35:58.234 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.44434.log: /var/log/ceph/cluster1-client.mirror.67285.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67285.log.gz 2024-09-18T01:35:58.234 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54252.log 2024-09-18T01:35:58.234 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56605.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.56605.log.gz 2024-09-18T01:35:58.237 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44348.log 2024-09-18T01:35:58.238 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54252.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44434.log.gz 2024-09-18T01:35:58.239 INFO:teuthology.orchestra.run.smithi150.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54252.log.gz 2024-09-18T01:35:58.241 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39688.log 2024-09-18T01:35:58.241 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.44348.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54701.log 2024-09-18T01:35:58.242 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39688.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51855.log 2024-09-18T01:35:58.242 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54701.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54701.log.gz 2024-09-18T01:35:58.243 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44858.log 2024-09-18T01:35:58.249 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51855.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44348.log.gz 2024-09-18T01:35:58.249 INFO:teuthology.orchestra.run.smithi150.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51855.log.gz 2024-09-18T01:35:58.249 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70566.log.gz 2024-09-18T01:35:58.249 INFO:teuthology.orchestra.run.smithi150.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.54007.log 2024-09-18T01:35:58.249 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44858.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51671.log 2024-09-18T01:35:58.249 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44858.log.gz 2024-09-18T01:35:58.249 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54007.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42517.log.gz 2024-09-18T01:35:58.249 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46967.log 2024-09-18T01:35:58.249 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51671.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50138.log 2024-09-18T01:35:58.249 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46967.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46967.log.gz 2024-09-18T01:35:58.250 INFO:teuthology.orchestra.run.smithi150.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54007.log.gz 2024-09-18T01:35:58.251 INFO:teuthology.orchestra.run.smithi150.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51671.log.gz 2024-09-18T01:35:58.251 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44878.log 2024-09-18T01:35:58.251 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51251.log 2024-09-18T01:35:58.251 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50138.log: /var/log/ceph/cluster2-client.mirror.44878.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43239.log 2024-09-18T01:35:58.253 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51251.log: 90.9% -- replaced with /var/log/ceph/cluster2-client.mirror.44878.log.gz 2024-09-18T01:35:58.253 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37285.log 2024-09-18T01:35:58.254 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43239.log: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.43239.log.gz 2024-09-18T01:35:58.255 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53786.log 2024-09-18T01:35:58.255 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.37285.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42111.log 2024-09-18T01:35:58.255 INFO:teuthology.orchestra.run.smithi150.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51251.log.gz 2024-09-18T01:35:58.256 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53786.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59168.log 2024-09-18T01:35:58.257 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.42111.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37285.log.gz 2024-09-18T01:35:58.259 INFO:teuthology.orchestra.run.smithi150.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53786.log.gz 2024-09-18T01:35:58.259 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59188.log 2024-09-18T01:35:58.259 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59168.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68587.log 2024-09-18T01:35:58.259 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59188.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59168.log.gz 2024-09-18T01:35:58.260 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59188.log.gz 2024-09-18T01:35:58.260 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50570.log 2024-09-18T01:35:58.261 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62059.log 2024-09-18T01:35:58.261 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50570.log: /var/log/ceph/cluster2-client.mirror.68587.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.68587.log.gz 2024-09-18T01:35:58.262 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.37022.log 2024-09-18T01:35:58.262 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.50570.log.gz 2024-09-18T01:35:58.262 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62059.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62059.log.gz 2024-09-18T01:35:58.264 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55565.log 2024-09-18T01:35:58.264 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49635.log 2024-09-18T01:35:58.264 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.0.37022.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.42111.log.gz 2024-09-18T01:35:58.264 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43441.log 2024-09-18T01:35:58.264 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55565.log: /var/log/ceph/cluster2-client.mirror.49635.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49635.log.gz 2024-09-18T01:35:58.265 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46021.log 2024-09-18T01:35:58.265 INFO:teuthology.orchestra.run.smithi150.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55565.log.gz 2024-09-18T01:35:58.267 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43441.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.39688.log.gz 2024-09-18T01:35:58.267 INFO:teuthology.orchestra.run.smithi150.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43441.log.gz 2024-09-18T01:35:58.267 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56400.log 2024-09-18T01:35:58.267 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46021.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59754.log 2024-09-18T01:35:58.267 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56400.log: 87.3% 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56400.log.gz 2024-09-18T01:35:58.267 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46021.log.gz 2024-09-18T01:35:58.267 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49422.log 2024-09-18T01:35:58.267 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51138.log 2024-09-18T01:35:58.271 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49422.log: /var/log/ceph/cluster1-client.mirror.59754.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59754.log.gz 2024-09-18T01:35:58.271 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54746.log 2024-09-18T01:35:58.275 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51138.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51138.log.gz 2024-09-18T01:35:58.276 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66407.log 2024-09-18T01:35:58.278 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54746.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54746.log.gz 2024-09-18T01:35:58.279 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66407.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56511.log 2024-09-18T01:35:58.279 INFO:teuthology.orchestra.run.smithi150.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66407.log.gz 2024-09-18T01:35:58.280 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.69426.log 2024-09-18T01:35:58.280 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56511.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56511.log.gz 2024-09-18T01:35:58.281 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49694.log.gz 2024-09-18T01:35:58.281 INFO:teuthology.orchestra.run.smithi150.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.37022.log.gz 2024-09-18T01:35:58.282 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41561.log 2024-09-18T01:35:58.283 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.0.69426.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.69426.log.gz 2024-09-18T01:35:58.283 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51626.log 2024-09-18T01:35:58.285 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41561.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44070.log 2024-09-18T01:35:58.285 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51626.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38889.log 2024-09-18T01:35:58.285 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44070.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51626.log.gz 2024-09-18T01:35:58.286 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64989.log 2024-09-18T01:35:58.287 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.38889.log: 90.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38889.log.gz 2024-09-18T01:35:58.288 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50138.log.gz 2024-09-18T01:35:58.291 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44070.log.gz 2024-09-18T01:35:58.299 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57384.log 2024-09-18T01:35:58.299 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64989.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46121.log 2024-09-18T01:35:58.299 INFO:teuthology.orchestra.run.smithi150.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64989.log.gz 2024-09-18T01:35:58.299 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57384.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64678.log 2024-09-18T01:35:58.299 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46121.log: 87.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54111.log 2024-09-18T01:35:58.299 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.57384.log.gz 2024-09-18T01:35:58.299 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46121.log.gz 2024-09-18T01:35:58.299 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58939.log 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64678.log: /var/log/ceph/cluster2-client.mirror.54111.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44697.log 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58939.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54111.log.gz 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64678.log.gz 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58939.log.gz 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49178.log 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44697.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59983.log 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44697.log.gz 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45368.log 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.42202.log 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59983.log: /var/log/ceph/cluster2-client.mirror.45368.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.59983.log.gz 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53299.log 2024-09-18T01:35:58.300 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.3.42202.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42059.log 2024-09-18T01:35:58.301 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.2.41331.log.gz 2024-09-18T01:35:58.301 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53299.log: /var/log/ceph/cluster2-client.mirror.49178.log: 87.9% 79.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53299.log.gz 2024-09-18T01:35:58.301 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.49178.log.gz 2024-09-18T01:35:58.301 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41468.log 2024-09-18T01:35:58.302 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69348.log 2024-09-18T01:35:58.303 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41468.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39012.log 2024-09-18T01:35:58.303 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69348.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69348.log.gz 2024-09-18T01:35:58.303 INFO:teuthology.orchestra.run.smithi150.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42059.log.gz 2024-09-18T01:35:58.304 INFO:teuthology.orchestra.run.smithi150.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41468.log.gz 2024-09-18T01:35:58.304 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51468.log 2024-09-18T01:35:58.304 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39012.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38964.log 2024-09-18T01:35:58.307 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44199.log 2024-09-18T01:35:58.307 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.38964.log: /var/log/ceph/cluster1-client.mirror.51468.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47962.log 2024-09-18T01:35:58.307 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44199.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.39012.log.gz 2024-09-18T01:35:58.308 INFO:teuthology.orchestra.run.smithi150.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51468.log.gz 2024-09-18T01:35:58.308 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60516.log 2024-09-18T01:35:58.308 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47962.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38964.log.gz 2024-09-18T01:35:58.321 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44199.log.gz 2024-09-18T01:35:58.323 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53938.log 2024-09-18T01:35:58.323 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60516.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.60516.log.gz 2024-09-18T01:35:58.323 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50391.log 2024-09-18T01:35:58.324 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53938.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53938.log.gz 2024-09-18T01:35:58.324 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48266.log 2024-09-18T01:35:58.325 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50391.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50391.log.gz 2024-09-18T01:35:58.325 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43984.log 2024-09-18T01:35:58.329 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48266.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48266.log.gz 2024-09-18T01:35:58.330 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66520.log 2024-09-18T01:35:58.342 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43984.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43984.log.gz 2024-09-18T01:35:58.344 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54677.log 2024-09-18T01:35:58.344 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66520.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66520.log.gz 2024-09-18T01:35:58.344 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58517.log 2024-09-18T01:35:58.344 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54677.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54677.log.gz 2024-09-18T01:35:58.346 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69327.log 2024-09-18T01:35:58.346 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58517.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.58517.log.gz 2024-09-18T01:35:58.346 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37572.log 2024-09-18T01:35:58.350 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.69327.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41561.log.gz 2024-09-18T01:35:58.350 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69327.log.gz 2024-09-18T01:35:58.352 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62784.log 2024-09-18T01:35:58.352 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.37572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41252.log 2024-09-18T01:35:58.352 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62784.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62784.log.gz 2024-09-18T01:35:58.354 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54725.log 2024-09-18T01:35:58.354 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41252.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54432.log 2024-09-18T01:35:58.354 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54725.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54725.log.gz 2024-09-18T01:35:58.354 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66610.log 2024-09-18T01:35:58.356 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54432.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54432.log.gz 2024-09-18T01:35:58.356 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44328.log 2024-09-18T01:35:58.357 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66610.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66610.log.gz 2024-09-18T01:35:58.357 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67195.log 2024-09-18T01:35:58.360 INFO:teuthology.orchestra.run.smithi150.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41252.log.gz 2024-09-18T01:35:58.361 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44328.log: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.37572.log.gz 2024-09-18T01:35:58.369 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44328.log.gz 2024-09-18T01:35:58.371 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41294.log 2024-09-18T01:35:58.372 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67195.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67195.log.gz 2024-09-18T01:35:58.372 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45814.log 2024-09-18T01:35:58.372 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41294.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35831.log 2024-09-18T01:35:58.372 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41294.log.gz 2024-09-18T01:35:58.428 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45814.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71150.log 2024-09-18T01:35:58.428 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.35831.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35831.log.gz 2024-09-18T01:35:58.437 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57931.log 2024-09-18T01:35:58.448 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71150.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45545.log 2024-09-18T01:35:58.449 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57931.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57931.log.gz 2024-09-18T01:35:58.450 INFO:teuthology.orchestra.run.smithi150.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71150.log.gz 2024-09-18T01:35:58.457 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46303.log.gz 2024-09-18T01:35:58.458 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42899.log 2024-09-18T01:35:58.479 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45545.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65009.log 2024-09-18T01:35:58.480 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42899.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43703.log 2024-09-18T01:35:58.481 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65009.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42899.log.gz 2024-09-18T01:35:58.481 INFO:teuthology.orchestra.run.smithi150.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65009.log.gz 2024-09-18T01:35:58.493 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60765.log 2024-09-18T01:35:58.497 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43703.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43703.log.gz 2024-09-18T01:35:58.509 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48910.log.gz 2024-09-18T01:35:58.510 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68971.log 2024-09-18T01:35:58.511 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60765.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60765.log.gz 2024-09-18T01:35:58.524 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40435.log 2024-09-18T01:35:58.524 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68971.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.68971.log.gz 2024-09-18T01:35:58.542 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57244.log 2024-09-18T01:35:58.554 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40435.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48030.log 2024-09-18T01:35:58.555 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57244.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.40435.log.gz 2024-09-18T01:35:58.555 INFO:teuthology.orchestra.run.smithi150.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.57244.log.gz 2024-09-18T01:35:58.564 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66677.log 2024-09-18T01:35:58.565 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48030.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48030.log.gz 2024-09-18T01:35:58.578 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37989.log 2024-09-18T01:35:58.579 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66677.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66677.log.gz 2024-09-18T01:35:58.587 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49775.log 2024-09-18T01:35:58.605 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37989.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37989.log.gz 2024-09-18T01:35:58.605 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68950.log 2024-09-18T01:35:58.615 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49775.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43031.log 2024-09-18T01:35:58.615 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.68950.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68950.log.gz 2024-09-18T01:35:58.623 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71763.log 2024-09-18T01:35:58.633 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43031.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47186.log 2024-09-18T01:35:58.634 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71763.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47804.log.gz 2024-09-18T01:35:58.643 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43031.log.gz 2024-09-18T01:35:58.644 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64006.log 2024-09-18T01:35:58.645 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47186.log: 86.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71763.log.gz 2024-09-18T01:35:58.659 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36122.log 2024-09-18T01:35:58.660 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64006.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64006.log.gz 2024-09-18T01:35:58.669 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48789.log 2024-09-18T01:35:58.678 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36122.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54294.log 2024-09-18T01:35:58.681 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48789.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36122.log.gz 2024-09-18T01:35:58.691 INFO:teuthology.orchestra.run.smithi150.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48789.log.gz 2024-09-18T01:35:58.694 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51879.log 2024-09-18T01:35:58.695 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54294.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54294.log.gz 2024-09-18T01:35:58.704 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48052.log.gz 2024-09-18T01:35:58.704 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55119.log 2024-09-18T01:35:58.706 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51879.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49422.log.gz 2024-09-18T01:35:58.709 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69890.log 2024-09-18T01:35:58.709 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55119.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55119.log.gz 2024-09-18T01:35:58.709 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45086.log 2024-09-18T01:35:58.709 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69890.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69890.log.gz 2024-09-18T01:35:58.710 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68771.log 2024-09-18T01:35:58.710 INFO:teuthology.orchestra.run.smithi150.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51879.log.gz 2024-09-18T01:35:58.721 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45086.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68501.log 2024-09-18T01:35:58.722 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68771.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.68771.log.gz 2024-09-18T01:35:58.736 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.37221.log 2024-09-18T01:35:58.745 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68501.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68501.log.gz 2024-09-18T01:35:58.747 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70063.log 2024-09-18T01:35:58.764 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.1.37221.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58377.log 2024-09-18T01:35:58.764 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.70063.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70063.log.gz 2024-09-18T01:35:58.781 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68545.log 2024-09-18T01:35:58.784 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58377.log: 92.2% 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.1.37059.log.gz 2024-09-18T01:35:58.788 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67600.log 2024-09-18T01:35:58.788 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.68545.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68545.log.gz 2024-09-18T01:35:58.788 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.40838.log 2024-09-18T01:35:58.788 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67600.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67600.log.gz 2024-09-18T01:35:58.788 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64533.log 2024-09-18T01:35:58.788 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.0.40838.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53103.log 2024-09-18T01:35:58.789 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64533.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64533.log.gz 2024-09-18T01:35:58.790 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57017.log 2024-09-18T01:35:58.794 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53103.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53103.log.gz 2024-09-18T01:35:58.794 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64318.log 2024-09-18T01:35:58.795 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57017.log: -- replaced with /var/log/ceph/cluster1-client.mirror.58377.log.gz 2024-09-18T01:35:58.796 INFO:teuthology.orchestra.run.smithi150.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57017.log.gz 2024-09-18T01:35:58.797 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66700.log 2024-09-18T01:35:58.802 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64318.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64318.log.gz 2024-09-18T01:35:58.804 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42947.log 2024-09-18T01:35:58.805 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66700.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66700.log.gz 2024-09-18T01:35:58.805 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62576.log 2024-09-18T01:35:58.805 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42947.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43011.log 2024-09-18T01:35:58.805 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62576.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62576.log.gz 2024-09-18T01:35:58.806 INFO:teuthology.orchestra.run.smithi150.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42947.log.gz 2024-09-18T01:35:58.807 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45044.log 2024-09-18T01:35:58.807 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43011.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37794.log 2024-09-18T01:35:58.807 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45044.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45044.log.gz 2024-09-18T01:35:58.817 INFO:teuthology.orchestra.run.smithi150.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.3.42202.log.gz 2024-09-18T01:35:58.818 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43011.log.gz 2024-09-18T01:35:58.820 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69975.log 2024-09-18T01:35:58.821 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.37794.log: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.40838.log.gz 2024-09-18T01:35:58.829 INFO:teuthology.orchestra.run.smithi150.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.1.37221.log.gz 2024-09-18T01:35:58.831 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64574.log 2024-09-18T01:35:58.831 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.69975.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.69975.log.gz 2024-09-18T01:35:58.831 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62176.log 2024-09-18T01:35:58.833 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64574.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64574.log.gz 2024-09-18T01:35:58.834 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65361.log 2024-09-18T01:35:58.835 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62176.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62176.log.gz 2024-09-18T01:35:58.838 INFO:teuthology.orchestra.run.smithi150.stderr:gzip 80.0% -- replaced with /var/log/ceph/cluster2-client.admin.37794.log.gz 2024-09-18T01:35:58.838 INFO:teuthology.orchestra.run.smithi150.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60540.log 2024-09-18T01:35:58.839 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65361.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65361.log.gz 2024-09-18T01:35:58.839 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45814.log.gz 2024-09-18T01:35:58.839 INFO:teuthology.orchestra.run.smithi150.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.61334.log 2024-09-18T01:35:58.839 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55364.log 2024-09-18T01:35:58.839 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60540.log: /var/log/ceph/cluster2-client.mirror.61334.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60540.log.gz 2024-09-18T01:35:58.843 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61334.log.gz 2024-09-18T01:35:58.843 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39937.log 2024-09-18T01:35:58.843 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55364.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64595.log 2024-09-18T01:35:58.843 INFO:teuthology.orchestra.run.smithi150.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55364.log.gz 2024-09-18T01:35:58.843 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61155.log 2024-09-18T01:35:58.843 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.64595.log: -- /var/log/ceph/cluster2-client.mirror.59437.log 2024-09-18T01:35:58.843 INFO:teuthology.orchestra.run.smithi150.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39937.log.gz 2024-09-18T01:35:58.844 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61155.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64595.log.gz 2024-09-18T01:35:58.844 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61155.log.gz 2024-09-18T01:35:58.844 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60900.log 2024-09-18T01:35:58.846 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59437.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59437.log.gz 2024-09-18T01:35:58.846 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47903.log 2024-09-18T01:35:58.846 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60900.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60900.log.gz 2024-09-18T01:35:58.846 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60076.log 2024-09-18T01:35:58.847 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47903.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47903.log.gz 2024-09-18T01:35:58.848 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64298.log 2024-09-18T01:35:58.850 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60076.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60076.log.gz 2024-09-18T01:35:58.850 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57083.log 2024-09-18T01:35:58.851 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64298.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64298.log.gz 2024-09-18T01:35:58.852 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51900.log 2024-09-18T01:35:58.853 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57083.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57083.log.gz 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45545.log.gz 2024-09-18T01:35:58.853 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:58.854 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52576.log 2024-09-18T01:35:58.854 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59557.log 2024-09-18T01:35:58.863 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51900.log: /var/log/ceph/cluster1-client.mirror.52576.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52576.log.gz 2024-09-18T01:35:58.863 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41994.log 2024-09-18T01:35:58.863 INFO:teuthology.orchestra.run.smithi150.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51900.log.gz 2024-09-18T01:35:58.863 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59557.log: gzip -5 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59557.log.gz 2024-09-18T01:35:58.863 INFO:teuthology.orchestra.run.smithi150.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.48454.log 2024-09-18T01:35:58.863 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56127.log 2024-09-18T01:35:58.863 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41994.log: /var/log/ceph/cluster2-client.mirror.48454.log: gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2024-09-18T01:35:58.864 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56127.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48454.log.gz 2024-09-18T01:35:58.864 INFO:teuthology.orchestra.run.smithi150.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.56127.log.gz 2024-09-18T01:35:58.864 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70701.log 2024-09-18T01:35:58.864 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41974.log 2024-09-18T01:35:58.864 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2.audit.log: /var/log/ceph/cluster2-client.mirror.70701.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40838.log 2024-09-18T01:35:58.864 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70701.log.gz 2024-09-18T01:35:58.864 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41974.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58469.log 2024-09-18T01:35:58.864 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40838.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54830.log 2024-09-18T01:35:58.864 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.58469.log: -- /var/log/ceph/cluster1-client.mirror.56199.log 2024-09-18T01:35:58.864 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58469.log.gz 2024-09-18T01:35:58.865 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47164.log 2024-09-18T01:35:58.865 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48950.log 82.0% -- replaced with /var/log/ceph/cluster1-client.mirror.40838.log.gz 2024-09-18T01:35:58.865 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:58.865 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47164.log: /var/log/ceph/cluster1-client.mirror.56199.log: 87.8% 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56199.log.gz 2024-09-18T01:35:58.865 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54830.log.gz 2024-09-18T01:35:58.865 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.47164.log.gz 2024-09-18T01:35:58.865 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56448.log 2024-09-18T01:35:58.865 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65609.log 2024-09-18T01:35:58.865 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48950.log: /var/log/ceph/cluster1-client.mirror.56448.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45567.log 2024-09-18T01:35:58.865 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65609.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.56448.log.gz 2024-09-18T01:35:58.865 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65609.log.gz 2024-09-18T01:35:58.868 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45567.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45567.log.gz 2024-09-18T01:35:58.868 INFO:teuthology.orchestra.run.smithi150.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48950.log.gz 2024-09-18T01:35:58.868 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68906.log 2024-09-18T01:35:58.868 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53569.log 2024-09-18T01:35:58.868 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68906.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65934.log 2024-09-18T01:35:58.868 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53569.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68906.log.gz 2024-09-18T01:35:58.868 INFO:teuthology.orchestra.run.smithi150.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53569.log.gz 2024-09-18T01:35:58.868 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57107.log 2024-09-18T01:35:58.869 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.65934.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65934.log.gz 2024-09-18T01:35:58.872 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63964.log 2024-09-18T01:35:58.872 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57107.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69927.log 2024-09-18T01:35:58.876 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63964.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41994.log.gz 2024-09-18T01:35:58.876 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63964.log.gz 2024-09-18T01:35:58.876 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.57107.log.gz 2024-09-18T01:35:58.877 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69012.log 2024-09-18T01:35:58.877 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69927.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46615.log 2024-09-18T01:35:58.877 INFO:teuthology.orchestra.run.smithi150.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69927.log.gz 2024-09-18T01:35:58.877 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69012.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41974.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54049.log 2024-09-18T01:35:58.877 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:58.877 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46615.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51763.log 2024-09-18T01:35:58.880 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54049.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46615.log.gz 2024-09-18T01:35:58.880 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54049.log.gz 2024-09-18T01:35:58.880 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68793.log 2024-09-18T01:35:58.881 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51763.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69012.log.gz 2024-09-18T01:35:58.881 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51763.log.gz 2024-09-18T01:35:58.881 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63737.log 2024-09-18T01:35:58.881 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68793.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50251.log 2024-09-18T01:35:58.881 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63737.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63737.log.gz 2024-09-18T01:35:58.884 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 92.8% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2024-09-18T01:35:58.884 INFO:teuthology.orchestra.run.smithi150.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.50933.log 2024-09-18T01:35:58.884 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.70245.log 2024-09-18T01:35:58.884 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50251.log: /var/log/ceph/cluster1-client.mirror.50933.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50933.log.gz 2024-09-18T01:35:58.884 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50251.log.gz 2024-09-18T01:35:58.884 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54946.log 2024-09-18T01:35:58.884 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.0.70245.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47683.log 2024-09-18T01:35:58.884 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54946.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54946.log.gz 2024-09-18T01:35:58.884 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62701.log 2024-09-18T01:35:58.886 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47683.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47683.log.gz 2024-09-18T01:35:58.886 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36374.log 2024-09-18T01:35:58.886 INFO:teuthology.orchestra.run.smithi150.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.70245.log.gz 2024-09-18T01:35:58.887 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62701.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62701.log.gz 2024-09-18T01:35:58.887 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67937.log 2024-09-18T01:35:58.888 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36374.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36374.log.gz 2024-09-18T01:35:58.888 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.46694.log 2024-09-18T01:35:58.888 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67937.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.67937.log.gz 2024-09-18T01:35:58.889 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63321.log 2024-09-18T01:35:58.907 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.1.46694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63923.log 2024-09-18T01:35:58.907 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63321.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63321.log.gz 2024-09-18T01:35:58.910 INFO:teuthology.orchestra.run.smithi150.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.1.46694.log.gz 2024-09-18T01:35:58.910 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69304.log 2024-09-18T01:35:58.912 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63923.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63923.log.gz 2024-09-18T01:35:58.912 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63383.log 2024-09-18T01:35:58.912 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.69304.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69304.log.gz 2024-09-18T01:35:58.914 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50616.log 2024-09-18T01:35:58.914 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63383.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63383.log.gz 2024-09-18T01:35:58.915 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49513.log 2024-09-18T01:35:58.916 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50616.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50616.log.gz 2024-09-18T01:35:58.917 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72006.log 2024-09-18T01:35:58.917 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49513.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49513.log.gz 2024-09-18T01:35:58.920 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55436.log 2024-09-18T01:35:58.920 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.72006.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.72006.log.gz 2024-09-18T01:35:58.920 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63675.log 2024-09-18T01:35:58.920 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55436.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.55436.log.gz 2024-09-18T01:35:58.920 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61551.log 2024-09-18T01:35:58.921 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63675.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63675.log.gz 2024-09-18T01:35:58.922 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67060.log 2024-09-18T01:35:58.922 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61551.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61551.log.gz 2024-09-18T01:35:58.924 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67892.log 2024-09-18T01:35:58.924 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67060.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67060.log.gz 2024-09-18T01:35:58.924 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57681.log 2024-09-18T01:35:58.929 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67892.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67892.log.gz 2024-09-18T01:35:58.929 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67645.log 2024-09-18T01:35:58.930 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57681.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57681.log.gz 2024-09-18T01:35:58.930 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44557.log 2024-09-18T01:35:58.930 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67645.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67645.log.gz 2024-09-18T01:35:58.930 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62312.log 2024-09-18T01:35:58.930 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44557.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44557.log.gz 2024-09-18T01:35:58.930 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44965.log 2024-09-18T01:35:58.931 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.62312.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.62312.log.gz 2024-09-18T01:35:58.931 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71960.log 2024-09-18T01:35:58.932 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44965.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.44965.log.gz 2024-09-18T01:35:58.934 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64760.log 2024-09-18T01:35:58.934 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71960.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71960.log.gz 2024-09-18T01:35:58.934 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51379.log 2024-09-18T01:35:58.937 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64760.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64760.log.gz 2024-09-18T01:35:58.937 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36503.log 2024-09-18T01:35:58.938 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51379.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51379.log.gz 2024-09-18T01:35:58.938 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57722.log 2024-09-18T01:35:58.938 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36503.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36503.log.gz 2024-09-18T01:35:58.940 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.37099.log 2024-09-18T01:35:58.942 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57722.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.57722.log.gz 2024-09-18T01:35:58.942 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69050.log 2024-09-18T01:35:58.943 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.2.37099.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71240.log 2024-09-18T01:35:58.948 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69050.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.2.37099.log.gz 2024-09-18T01:35:58.949 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69050.log.gz 2024-09-18T01:35:58.949 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41047.log 2024-09-18T01:35:58.950 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71240.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.71240.log.gz 2024-09-18T01:35:58.950 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47144.log 2024-09-18T01:35:58.951 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41047.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61075.log 2024-09-18T01:35:58.952 INFO:teuthology.orchestra.run.smithi150.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.41047.log.gz 2024-09-18T01:35:58.953 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47144.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47144.log.gz 2024-09-18T01:35:58.953 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51162.log 2024-09-18T01:35:58.956 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61075.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61075.log.gz 2024-09-18T01:35:58.956 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40759.log 2024-09-18T01:35:58.957 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51162.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.51162.log.gz 2024-09-18T01:35:58.957 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43575.log 2024-09-18T01:35:58.958 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40759.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46202.log 2024-09-18T01:35:58.960 INFO:teuthology.orchestra.run.smithi150.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40759.log.gz 2024-09-18T01:35:58.961 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43575.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.43575.log.gz 2024-09-18T01:35:58.961 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54384.log 2024-09-18T01:35:58.961 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46202.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46202.log.gz 2024-09-18T01:35:58.962 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58814.log 2024-09-18T01:35:58.963 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54384.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54384.log.gz 2024-09-18T01:35:58.964 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50228.log 2024-09-18T01:35:58.965 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58814.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58814.log.gz 2024-09-18T01:35:58.965 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47962.log.gz 2024-09-18T01:35:58.966 INFO:teuthology.orchestra.run.smithi150.stderr: -- /var/log/ceph/cluster1-client.mirror.60359.log 2024-09-18T01:35:58.966 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44305.log 2024-09-18T01:35:58.966 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50228.log: 2.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50228.log.gz 2024-09-18T01:35:58.967 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58674.log 2024-09-18T01:35:58.967 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.44305.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60359.log.gz 2024-09-18T01:35:58.967 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62825.log 2024-09-18T01:35:58.968 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58674.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58674.log.gz 2024-09-18T01:35:58.968 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63634.log 2024-09-18T01:35:58.969 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62825.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.62825.log.gz 2024-09-18T01:35:58.969 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36375.log 2024-09-18T01:35:58.970 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63634.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63634.log.gz 2024-09-18T01:35:58.970 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47883.log 2024-09-18T01:35:58.971 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36375.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36375.log.gz 2024-09-18T01:35:58.972 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69389.log 2024-09-18T01:35:58.972 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44305.log.gz/var/log/ceph/cluster2-client.mirror.47883.log: 2024-09-18T01:35:58.973 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58541.log 2024-09-18T01:35:58.975 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69389.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69389.log.gz 2024-09-18T01:35:58.976 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36249.log 2024-09-18T01:35:58.979 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58541.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.58541.log.gz 2024-09-18T01:35:58.979 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47940.log 2024-09-18T01:35:58.979 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36249.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36249.log.gz 2024-09-18T01:35:58.979 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62949.log 2024-09-18T01:35:58.979 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47940.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47940.log.gz 2024-09-18T01:35:58.979 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44391.log 2024-09-18T01:35:58.980 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62949.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62949.log.gz 2024-09-18T01:35:58.980 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52460.log 2024-09-18T01:35:58.986 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.44391.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44391.log.gz 2024-09-18T01:35:58.986 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68168.log 2024-09-18T01:35:58.988 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52460.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52460.log.gz 2024-09-18T01:35:58.988 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66992.log 2024-09-18T01:35:59.014 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68168.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58493.log 2024-09-18T01:35:59.015 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66992.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66992.log.gz 2024-09-18T01:35:59.029 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57838.log 2024-09-18T01:35:59.029 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58493.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.58493.log.gz 2024-09-18T01:35:59.033 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45368.log.gz 2024-09-18T01:35:59.033 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49733.log 2024-09-18T01:35:59.034 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57838.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.57838.log.gz 2024-09-18T01:35:59.034 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61392.log 2024-09-18T01:35:59.036 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49733.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49733.log.gz 2024-09-18T01:35:59.036 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71217.log 2024-09-18T01:35:59.038 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61392.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61392.log.gz 2024-09-18T01:35:59.038 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58606.log 2024-09-18T01:35:59.038 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71217.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.71217.log.gz 2024-09-18T01:35:59.038 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57336.log 2024-09-18T01:35:59.040 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58606.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.58606.log.gz 2024-09-18T01:35:59.040 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66160.log 2024-09-18T01:35:59.043 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57336.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57336.log.gz 2024-09-18T01:35:59.043 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69701.log 2024-09-18T01:35:59.043 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66160.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66160.log.gz 2024-09-18T01:35:59.043 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71027.log 2024-09-18T01:35:59.044 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.69701.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69701.log.gz 2024-09-18T01:35:59.047 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53962.log 2024-09-18T01:35:59.047 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71027.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.71027.log.gz 2024-09-18T01:35:59.047 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48727.log 2024-09-18T01:35:59.047 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53962.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53962.log.gz 2024-09-18T01:35:59.048 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56059.log 2024-09-18T01:35:59.048 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48727.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48727.log.gz 2024-09-18T01:35:59.049 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59461.log 2024-09-18T01:35:59.050 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56059.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56059.log.gz 2024-09-18T01:35:59.050 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57702.log 2024-09-18T01:35:59.051 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59461.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59461.log.gz 2024-09-18T01:35:59.051 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45604.log 2024-09-18T01:35:59.054 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57702.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57702.log.gz 2024-09-18T01:35:59.054 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72238.log 2024-09-18T01:35:59.054 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45604.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45604.log.gz 2024-09-18T01:35:59.056 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61117.log 2024-09-18T01:35:59.056 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72238.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72238.log.gz 2024-09-18T01:35:59.056 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59843.log 2024-09-18T01:35:59.058 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61117.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61117.log.gz 2024-09-18T01:35:59.058 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41927.log 2024-09-18T01:35:59.061 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59843.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59843.log.gz 2024-09-18T01:35:59.061 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59959.log 2024-09-18T01:35:59.062 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41927.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57131.log 2024-09-18T01:35:59.062 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59959.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41927.log.gz 2024-09-18T01:35:59.062 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59959.log.gz 2024-09-18T01:35:59.063 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46984.log 2024-09-18T01:35:59.064 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57131.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.57131.log.gz 2024-09-18T01:35:59.064 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48556.log 2024-09-18T01:35:59.066 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46984.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46984.log.gz 2024-09-18T01:35:59.066 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66587.log 2024-09-18T01:35:59.077 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48556.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36123.log 2024-09-18T01:35:59.077 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66587.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66587.log.gz 2024-09-18T01:35:59.091 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59662.log 2024-09-18T01:35:59.092 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36123.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36123.log.gz 2024-09-18T01:35:59.092 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37725.log 2024-09-18T01:35:59.099 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59662.log: 89.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58011.log 2024-09-18T01:35:59.099 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.59662.log.gz 2024-09-18T01:35:59.100 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.37725.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.37725.log.gz 2024-09-18T01:35:59.113 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49832.log 2024-09-18T01:35:59.126 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58011.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54766.log 2024-09-18T01:35:59.126 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49832.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49832.log.gz 2024-09-18T01:35:59.138 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57155.log 2024-09-18T01:35:59.139 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54766.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.54766.log.gz 2024-09-18T01:35:59.147 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42923.log 2024-09-18T01:35:59.147 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57155.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58011.log.gz 2024-09-18T01:35:59.147 INFO:teuthology.orchestra.run.smithi150.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.57155.log.gz 2024-09-18T01:35:59.165 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69722.log 2024-09-18T01:35:59.169 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42923.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.42923.log.gz 2024-09-18T01:35:59.175 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47243.log 2024-09-18T01:35:59.175 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69722.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.69722.log.gz 2024-09-18T01:35:59.184 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58357.log 2024-09-18T01:35:59.197 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47243.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47243.log.gz 2024-09-18T01:35:59.201 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68091.log 2024-09-18T01:35:59.203 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58357.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.58357.log.gz 2024-09-18T01:35:59.210 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45626.log 2024-09-18T01:35:59.220 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68091.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64885.log 2024-09-18T01:35:59.223 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45626.log: 2.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68091.log.gz 2024-09-18T01:35:59.227 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68793.log.gz 2024-09-18T01:35:59.228 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43898.log 2024-09-18T01:35:59.230 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64885.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64885.log.gz 2024-09-18T01:35:59.230 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52624.log 2024-09-18T01:35:59.236 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43898.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43898.log.gz 2024-09-18T01:35:59.237 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59011.log 2024-09-18T01:35:59.238 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52624.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52624.log.gz 2024-09-18T01:35:59.238 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43918.log 2024-09-18T01:35:59.239 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59011.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59011.log.gz 2024-09-18T01:35:59.239 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51783.log 2024-09-18T01:35:59.245 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43918.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41879.log 2024-09-18T01:35:59.246 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51783.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51783.log.gz 2024-09-18T01:35:59.247 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44797.log 2024-09-18T01:35:59.252 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41879.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49775.log.gz 2024-09-18T01:35:59.252 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39773.log 2024-09-18T01:35:59.253 INFO:teuthology.orchestra.run.smithi150.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41879.log.gz 2024-09-18T01:35:59.256 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44797.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44797.log.gz 2024-09-18T01:35:59.256 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43918.log.gz 2024-09-18T01:35:59.256 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69998.log 2024-09-18T01:35:59.256 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39773.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57360.log 2024-09-18T01:35:59.256 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.69998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62405.log 2024-09-18T01:35:59.257 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57360.log: 87.7% 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39773.log.gz 2024-09-18T01:35:59.257 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.57360.log.gz 2024-09-18T01:35:59.257 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.69998.log.gz 2024-09-18T01:35:59.258 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41624.log 2024-09-18T01:35:59.259 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62405.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62405.log.gz 2024-09-18T01:35:59.259 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63778.log 2024-09-18T01:35:59.260 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41624.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52159.log 2024-09-18T01:35:59.263 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63778.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63778.log.gz 2024-09-18T01:35:59.265 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56742.log 2024-09-18T01:35:59.270 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52159.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52159.log.gz 2024-09-18T01:35:59.270 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41624.log.gz 2024-09-18T01:35:59.270 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55011.log 2024-09-18T01:35:59.270 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56742.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58790.log 2024-09-18T01:35:59.271 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55011.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55011.log.gz 2024-09-18T01:35:59.272 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54135.log 2024-09-18T01:35:59.273 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58790.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.58790.log.gz 2024-09-18T01:35:59.274 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57744.log 2024-09-18T01:35:59.274 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54135.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.54135.log.gz 2024-09-18T01:35:59.275 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67420.log 2024-09-18T01:35:59.276 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57744.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57744.log.gz 2024-09-18T01:35:59.277 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55681.log 2024-09-18T01:35:59.277 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67420.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67420.log.gz 2024-09-18T01:35:59.278 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45133.log 2024-09-18T01:35:59.278 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55681.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55681.log.gz 2024-09-18T01:35:59.279 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54653.log 2024-09-18T01:35:59.283 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45133.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.56742.log.gz 2024-09-18T01:35:59.287 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68927.log 2024-09-18T01:35:59.294 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44617.log 2024-09-18T01:35:59.295 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54653.log: /var/log/ceph/cluster1-client.mirror.68927.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.54653.log.gz 2024-09-18T01:35:59.295 INFO:teuthology.orchestra.run.smithi150.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.68927.log.gz 2024-09-18T01:35:59.307 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40459.log 2024-09-18T01:35:59.308 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44617.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.44617.log.gz 2024-09-18T01:35:59.315 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68480.log 2024-09-18T01:35:59.317 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40459.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.40459.log.gz 2024-09-18T01:35:59.330 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54091.log 2024-09-18T01:35:59.331 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.68480.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.68480.log.gz 2024-09-18T01:35:59.339 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63985.log 2024-09-18T01:35:59.340 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54091.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54091.log.gz 2024-09-18T01:35:59.348 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47105.log 2024-09-18T01:35:59.349 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63985.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63985.log.gz 2024-09-18T01:35:59.354 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47186.log.gz 2024-09-18T01:35:59.354 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43855.log 2024-09-18T01:35:59.357 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47105.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45133.log.gz 2024-09-18T01:35:59.357 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43287.log 2024-09-18T01:35:59.364 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43855.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43855.log.gz 2024-09-18T01:35:59.364 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64740.log 2024-09-18T01:35:59.364 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43287.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43287.log.gz 2024-09-18T01:35:59.379 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62763.log 2024-09-18T01:35:59.379 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64740.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64740.log.gz 2024-09-18T01:35:59.384 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69239.log 2024-09-18T01:35:59.384 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62763.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37555.log 2024-09-18T01:35:59.384 INFO:teuthology.orchestra.run.smithi150.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62763.log.gz 2024-09-18T01:35:59.384 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.69239.log: gzip 87.3% -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59620.log 2024-09-18T01:35:59.384 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.69239.log.gz 2024-09-18T01:35:59.385 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59682.log 2024-09-18T01:35:59.385 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.37555.log: /var/log/ceph/cluster2-client.mirror.59620.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71367.log 2024-09-18T01:35:59.385 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59682.log: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.37555.log.gz 2024-09-18T01:35:59.385 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59682.log.gz 2024-09-18T01:35:59.385 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59620.log.gz 2024-09-18T01:35:59.386 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64968.log 2024-09-18T01:35:59.388 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71367.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71367.log.gz 2024-09-18T01:35:59.388 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61641.log 2024-09-18T01:35:59.389 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64968.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64968.log.gz 2024-09-18T01:35:59.390 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44717.log 2024-09-18T01:35:59.391 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61641.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61641.log.gz 2024-09-18T01:35:59.391 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43875.log 2024-09-18T01:35:59.392 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44717.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.44717.log.gz 2024-09-18T01:35:59.392 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36881.log 2024-09-18T01:35:59.399 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43875.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43875.log.gz 2024-09-18T01:35:59.399 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48434.log 2024-09-18T01:35:59.399 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36881.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36881.log.gz 2024-09-18T01:35:59.400 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46143.log 2024-09-18T01:35:59.401 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48434.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48434.log.gz 2024-09-18T01:35:59.401 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46657.log 2024-09-18T01:35:59.414 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46143.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55256.log 2024-09-18T01:35:59.420 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46657.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62534.log 2024-09-18T01:35:59.424 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55256.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68168.log.gz 2024-09-18T01:35:59.426 INFO:teuthology.orchestra.run.smithi150.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55256.log.gz 2024-09-18T01:35:59.427 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40507.log 2024-09-18T01:35:59.428 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62534.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.62534.log.gz 2024-09-18T01:35:59.428 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67712.log 2024-09-18T01:35:59.429 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40507.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42081.log 2024-09-18T01:35:59.429 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67712.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67712.log.gz 2024-09-18T01:35:59.431 INFO:teuthology.orchestra.run.smithi150.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.40507.log.gz 2024-09-18T01:35:59.431 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57456.log 2024-09-18T01:35:59.431 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42081.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61754.log 2024-09-18T01:35:59.432 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57456.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57456.log.gz 2024-09-18T01:35:59.432 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51044.log 2024-09-18T01:35:59.435 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61754.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61754.log.gz 2024-09-18T01:35:59.435 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36545.log 2024-09-18T01:35:59.442 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51044.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51044.log.gz 2024-09-18T01:35:59.446 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64616.log 2024-09-18T01:35:59.446 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36545.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36545.log.gz 2024-09-18T01:35:59.447 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38988.log 2024-09-18T01:35:59.448 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64616.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64616.log.gz 2024-09-18T01:35:59.448 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47703.log 2024-09-18T01:35:59.448 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.38988.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65071.log 2024-09-18T01:35:59.449 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47703.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47703.log.gz 2024-09-18T01:35:59.450 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71641.log 2024-09-18T01:35:59.453 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.38988.log.gz 2024-09-18T01:35:59.455 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47883.log.gz 2024-09-18T01:35:59.458 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65071.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65071.log.gz 2024-09-18T01:35:59.466 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47635.log 2024-09-18T01:35:59.466 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62928.log 2024-09-18T01:35:59.469 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47635.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59260.log 2024-09-18T01:35:59.470 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71641.log: /var/log/ceph/cluster2-client.mirror.62928.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.71641.log.gz 2024-09-18T01:35:59.471 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62928.log.gz 2024-09-18T01:35:59.472 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66835.log 2024-09-18T01:35:59.472 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59260.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59260.log.gz 2024-09-18T01:35:59.480 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58766.log 2024-09-18T01:35:59.480 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66835.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66835.log.gz 2024-09-18T01:35:59.483 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44133.log 2024-09-18T01:35:59.483 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58766.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58766.log.gz 2024-09-18T01:35:59.483 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45086.log.gz 2024-09-18T01:35:59.483 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65133.log 2024-09-18T01:35:59.485 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.44133.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40576.log 2024-09-18T01:35:59.502 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65133.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65133.log.gz 2024-09-18T01:35:59.502 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2024-09-18T01:35:59.502 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40576.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50058.log 2024-09-18T01:35:59.502 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61006.log 2024-09-18T01:35:59.502 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50058.log: 2.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50058.log.gz 2024-09-18T01:35:59.502 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44133.log.gz 2024-09-18T01:35:59.502 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65237.log 2024-09-18T01:35:59.502 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61006.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39441.log 2024-09-18T01:35:59.502 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61006.log.gz 2024-09-18T01:35:59.502 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65237.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71326.log 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39441.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65237.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68734.log 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40185.log 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71326.log.gz 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55746.log 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40185.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68734.log.gz 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41199.log 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36248.log 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.41199.log: /var/log/ceph/cluster1-client.mirror.55746.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39441.log.gz 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55746.log.gz 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53745.log 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36248.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36248.log.gz 2024-09-18T01:35:59.503 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65588.log 2024-09-18T01:35:59.504 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53745.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.41199.log.gz 2024-09-18T01:35:59.504 INFO:teuthology.orchestra.run.smithi150.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53745.log.gz 2024-09-18T01:35:59.504 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63882.log 2024-09-18T01:35:59.504 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65588.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50548.log 2024-09-18T01:35:59.504 INFO:teuthology.orchestra.run.smithi150.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.65588.log.gz 2024-09-18T01:35:59.504 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63882.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63882.log.gz 2024-09-18T01:35:59.504 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49400.log 2024-09-18T01:35:59.504 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50548.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64347.log 2024-09-18T01:35:59.504 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49400.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49400.log.gz 2024-09-18T01:35:59.504 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53504.log 2024-09-18T01:35:59.515 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64347.log: 92.3% 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64347.log.gz 2024-09-18T01:35:59.515 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.50548.log.gz 2024-09-18T01:35:59.515 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72658.log 2024-09-18T01:35:59.515 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42405.log 2024-09-18T01:35:59.515 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72658.log: /var/log/ceph/cluster1-client.mirror.53504.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72658.log.gz 2024-09-18T01:35:59.515 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53504.log.gz 2024-09-18T01:35:59.515 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.48556.log.gz 2024-09-18T01:35:59.515 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65299.log 2024-09-18T01:35:59.515 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.42405.log: -- /var/log/ceph/cluster1-client.mirror.65956.log 2024-09-18T01:35:59.515 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67870.log 2024-09-18T01:35:59.515 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.65956.log: /var/log/ceph/cluster2-client.mirror.65299.log: gzip -5 --verbose 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.65956.log.gz 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr: -- /var/log/ceph/cluster2-client.mirror.48188.log 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67870.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59035.log 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67870.log.gz 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48188.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65299.log.gz 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66542.log 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48188.log.gz 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59035.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50528.log 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59035.log.gz 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52237.log 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66542.log: /var/log/ceph/cluster2-client.mirror.50528.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43195.log 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr: 86.8%/var/log/ceph/cluster2-client.mirror.52237.log: -- replaced with /var/log/ceph/cluster1-client.mirror.66542.log.gz 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64719.log 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52237.log.gz 2024-09-18T01:35:59.516 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43195.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50528.log.gz 2024-09-18T01:35:59.517 INFO:teuthology.orchestra.run.smithi150.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43195.log.gz 2024-09-18T01:35:59.517 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50100.log 2024-09-18T01:35:59.517 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39849.log 2024-09-18T01:35:59.517 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50100.log: /var/log/ceph/cluster2-client.mirror.64719.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50100.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49237.log 2024-09-18T01:35:59.517 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:35:59.517 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.39849.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64719.log.gz 2024-09-18T01:35:59.517 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.45133.log 2024-09-18T01:35:59.517 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40999.log 2024-09-18T01:35:59.517 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49237.log: /var/log/ceph/cluster1-client.mirror.2.45133.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46499.log 2024-09-18T01:35:59.520 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40999.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.40999.log.gz 2024-09-18T01:35:59.520 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57814.log 2024-09-18T01:35:59.520 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46499.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.39849.log.gz 2024-09-18T01:35:59.526 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40576.log.gz 2024-09-18T01:35:59.533 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62597.log 2024-09-18T01:35:59.534 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57814.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57814.log.gz 2024-09-18T01:35:59.542 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72574.log 2024-09-18T01:35:59.551 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62597.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.62597.log.gz 2024-09-18T01:35:59.554 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55191.log 2024-09-18T01:35:59.555 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72574.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72574.log.gz 2024-09-18T01:35:59.562 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71726.log 2024-09-18T01:35:59.562 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55191.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55191.log.gz 2024-09-18T01:35:59.575 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51273.log 2024-09-18T01:35:59.577 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71726.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71726.log.gz 2024-09-18T01:35:59.585 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52793.log 2024-09-18T01:35:59.599 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51273.log: 89.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40098.log 2024-09-18T01:35:59.599 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52793.log: -- replaced with /var/log/ceph/cluster2-client.mirror.51273.log.gz 2024-09-18T01:35:59.599 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52793.log.gz 2024-09-18T01:35:59.609 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43503.log 2024-09-18T01:35:59.620 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40098.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40163.log 2024-09-18T01:35:59.620 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43503.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43503.log.gz 2024-09-18T01:35:59.628 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.70937.log 2024-09-18T01:35:59.635 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40163.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.40098.log.gz 2024-09-18T01:35:59.643 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40163.log.gz 2024-09-18T01:35:59.644 INFO:teuthology.orchestra.run.smithi150.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.42405.log.gz 2024-09-18T01:35:59.644 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66632.log 2024-09-18T01:35:59.647 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.2.70937.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.2.70937.log.gz 2024-09-18T01:35:59.652 INFO:teuthology.orchestra.run.smithi150.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.45133.log.gz 2024-09-18T01:35:59.653 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41492.log 2024-09-18T01:35:59.656 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66632.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66632.log.gz 2024-09-18T01:35:59.656 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35996.log 2024-09-18T01:35:59.656 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41492.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37059.log 2024-09-18T01:35:59.656 INFO:teuthology.orchestra.run.smithi150.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41492.log.gz 2024-09-18T01:35:59.657 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.35996.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35996.log.gz 2024-09-18T01:35:59.658 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47365.log 2024-09-18T01:35:59.659 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57519.log 2024-09-18T01:35:59.659 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47365.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47365.log.gz 2024-09-18T01:35:59.660 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58565.log 2024-09-18T01:35:59.661 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57519.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57519.log.gz 2024-09-18T01:35:59.661 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38836.log 2024-09-18T01:35:59.663 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.58565.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.58565.log.gz 2024-09-18T01:35:59.663 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70507.log 2024-09-18T01:35:59.665 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.38836.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48809.log 2024-09-18T01:35:59.665 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70507.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70507.log.gz 2024-09-18T01:35:59.665 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44657.log 2024-09-18T01:35:59.666 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48809.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48809.log.gz 2024-09-18T01:35:59.666 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52482.log 2024-09-18T01:35:59.668 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44657.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44657.log.gz 2024-09-18T01:35:59.668 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41331.log 2024-09-18T01:35:59.670 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52482.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.38836.log.gz 2024-09-18T01:35:59.670 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52482.log.gz 2024-09-18T01:35:59.670 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52371.log 2024-09-18T01:35:59.670 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41331.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43335.log 2024-09-18T01:35:59.673 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52371.log: 78.0% -- replaced with /var/log/ceph/cluster2-client.mirror.41331.log.gz 2024-09-18T01:35:59.673 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52371.log.gz 2024-09-18T01:35:59.673 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48374.log 2024-09-18T01:35:59.673 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43335.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43335.log.gz 2024-09-18T01:35:59.673 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43153.log 2024-09-18T01:35:59.675 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48374.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48374.log.gz 2024-09-18T01:35:59.675 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64698.log 2024-09-18T01:35:59.677 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43153.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43153.log.gz 2024-09-18T01:35:59.677 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49753.log 2024-09-18T01:35:59.680 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64698.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64698.log.gz 2024-09-18T01:35:59.680 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45836.log 2024-09-18T01:35:59.682 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49753.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49753.log.gz 2024-09-18T01:35:59.682 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65712.log 2024-09-18T01:35:59.682 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45836.log: 79.3% -- replaced with /var/log/ceph/cluster2-client.mirror.45836.log.gz 2024-09-18T01:35:59.683 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56722.log 2024-09-18T01:35:59.684 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65712.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65712.log.gz 2024-09-18T01:35:59.684 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48593.log 2024-09-18T01:35:59.686 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56722.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56722.log.gz 2024-09-18T01:35:59.686 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41743.log 2024-09-18T01:35:59.693 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48593.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48593.log.gz 2024-09-18T01:35:59.693 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72532.log 2024-09-18T01:35:59.695 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.41743.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43941.log 2024-09-18T01:35:59.695 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72532.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72532.log.gz 2024-09-18T01:35:59.695 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42968.log 2024-09-18T01:35:59.696 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43941.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.41743.log.gz 2024-09-18T01:35:59.702 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43941.log.gz 2024-09-18T01:35:59.702 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65443.log 2024-09-18T01:35:59.703 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42968.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60286.log 2024-09-18T01:35:59.705 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65443.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65443.log.gz 2024-09-18T01:35:59.708 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64906.log 2024-09-18T01:35:59.709 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60286.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60286.log.gz 2024-09-18T01:35:59.709 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40706.log 2024-09-18T01:35:59.710 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64906.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64906.log.gz 2024-09-18T01:35:59.719 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42968.log.gz 2024-09-18T01:35:59.723 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70401.log 2024-09-18T01:35:59.724 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.40706.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52669.log 2024-09-18T01:35:59.724 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.70401.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.70401.log.gz 2024-09-18T01:35:59.728 INFO:teuthology.orchestra.run.smithi150.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.40706.log.gz 2024-09-18T01:35:59.728 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62492.log 2024-09-18T01:35:59.733 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52669.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52669.log.gz 2024-09-18T01:35:59.734 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70164.log 2024-09-18T01:35:59.742 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62492.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44047.log 2024-09-18T01:35:59.747 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70164.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62492.log.gz 2024-09-18T01:35:59.749 INFO:teuthology.orchestra.run.smithi150.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70164.log.gz 2024-09-18T01:35:59.753 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47046.log 2024-09-18T01:35:59.758 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.44047.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44047.log.gz 2024-09-18T01:35:59.761 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42017.log 2024-09-18T01:35:59.761 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47046.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47046.log.gz 2024-09-18T01:35:59.761 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54585.log 2024-09-18T01:35:59.761 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42017.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69949.log 2024-09-18T01:35:59.761 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54585.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54585.log.gz 2024-09-18T01:35:59.762 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41855.log 2024-09-18T01:35:59.772 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69949.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58835.log 2024-09-18T01:35:59.776 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41855.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41855.log.gz 2024-09-18T01:35:59.789 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45407.log 2024-09-18T01:35:59.791 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58835.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58835.log.gz 2024-09-18T01:35:59.795 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45626.log.gz 2024-09-18T01:35:59.797 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47206.log 2024-09-18T01:35:59.798 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45407.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45407.log.gz 2024-09-18T01:35:59.807 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60096.log 2024-09-18T01:35:59.807 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47206.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42017.log.gz 2024-09-18T01:35:59.810 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62336.log 2024-09-18T01:35:59.815 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60096.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47206.log.gz 2024-09-18T01:35:59.816 INFO:teuthology.orchestra.run.smithi150.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60096.log.gz 2024-09-18T01:35:59.817 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63135.log 2024-09-18T01:35:59.817 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.62336.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62336.log.gz 2024-09-18T01:35:59.818 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59865.log 2024-09-18T01:35:59.819 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63135.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63135.log.gz 2024-09-18T01:35:59.826 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60588.log 2024-09-18T01:35:59.826 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59865.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59865.log.gz 2024-09-18T01:35:59.826 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70184.log 2024-09-18T01:35:59.826 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60588.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60588.log.gz 2024-09-18T01:35:59.826 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44004.log 2024-09-18T01:35:59.826 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70184.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70184.log.gz 2024-09-18T01:35:59.826 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46800.log 2024-09-18T01:35:59.834 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.44004.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44004.log.gz 2024-09-18T01:35:59.834 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59578.log 2024-09-18T01:35:59.836 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46800.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46800.log.gz 2024-09-18T01:35:59.836 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66092.log 2024-09-18T01:35:59.846 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63176.log 2024-09-18T01:35:59.846 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66092.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66092.log.gz 2024-09-18T01:35:59.853 INFO:teuthology.orchestra.run.smithi150.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59578.log.gz 2024-09-18T01:35:59.854 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72054.log 2024-09-18T01:35:59.860 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63176.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63176.log.gz 2024-09-18T01:35:59.862 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43660.log 2024-09-18T01:35:59.862 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.72054.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.72054.log.gz 2024-09-18T01:35:59.862 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48151.log 2024-09-18T01:35:59.871 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43660.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43660.log.gz 2024-09-18T01:35:59.871 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53591.log 2024-09-18T01:35:59.871 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48151.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48151.log.gz 2024-09-18T01:35:59.872 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49380.log 2024-09-18T01:35:59.875 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53591.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53591.log.gz 2024-09-18T01:35:59.876 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64554.log 2024-09-18T01:35:59.877 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49380.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49380.log.gz 2024-09-18T01:35:59.877 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68027.log 2024-09-18T01:35:59.878 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64554.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64554.log.gz 2024-09-18T01:35:59.878 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44156.log 2024-09-18T01:35:59.879 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.68027.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68027.log.gz 2024-09-18T01:35:59.880 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67375.log 2024-09-18T01:35:59.889 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44156.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44156.log.gz 2024-09-18T01:35:59.889 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46182.log 2024-09-18T01:35:59.890 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67375.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67375.log.gz 2024-09-18T01:35:59.890 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41540.log 2024-09-18T01:35:59.891 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46182.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46182.log.gz 2024-09-18T01:35:59.893 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37751.log 2024-09-18T01:35:59.893 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41540.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64492.log 2024-09-18T01:35:59.893 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.37751.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.37751.log.gz 2024-09-18T01:35:59.894 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59533.log 2024-09-18T01:35:59.896 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64492.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64492.log.gz 2024-09-18T01:35:59.896 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57408.log 2024-09-18T01:35:59.897 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59533.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.59533.log.gz 2024-09-18T01:35:59.897 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63259.log 2024-09-18T01:35:59.897 INFO:teuthology.orchestra.run.smithi150.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.41540.log.gz 2024-09-18T01:35:59.899 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57408.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57408.log.gz 2024-09-18T01:35:59.899 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42453.log 2024-09-18T01:35:59.899 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63259.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63259.log.gz 2024-09-18T01:35:59.900 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47382.log 2024-09-18T01:35:59.901 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42453.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46902.log 2024-09-18T01:35:59.901 INFO:teuthology.orchestra.run.smithi150.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.42453.log.gz 2024-09-18T01:35:59.903 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47382.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47382.log.gz 2024-09-18T01:35:59.903 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50079.log 2024-09-18T01:35:59.906 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46902.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46902.log.gz 2024-09-18T01:35:59.906 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41131.log 2024-09-18T01:35:59.907 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50079.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50079.log.gz 2024-09-18T01:35:59.907 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47613.log 2024-09-18T01:35:59.908 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41131.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52966.log 2024-09-18T01:35:59.908 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47613.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47613.log.gz 2024-09-18T01:35:59.909 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51311.log 2024-09-18T01:35:59.910 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52966.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52966.log.gz 2024-09-18T01:35:59.912 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68751.log 2024-09-18T01:35:59.912 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51311.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51311.log.gz 2024-09-18T01:35:59.914 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61776.log 2024-09-18T01:35:59.916 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68751.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68751.log.gz 2024-09-18T01:35:59.916 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39232.log 2024-09-18T01:35:59.919 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61776.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61776.log.gz 2024-09-18T01:35:59.919 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32862.log 2024-09-18T01:35:59.920 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.39232.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54231.log 2024-09-18T01:35:59.921 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.32862.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32862.log.gz 2024-09-18T01:35:59.924 INFO:teuthology.orchestra.run.smithi150.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.39232.log.gz 2024-09-18T01:35:59.924 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64388.log 2024-09-18T01:35:59.925 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54231.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54231.log.gz 2024-09-18T01:35:59.928 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47841.log 2024-09-18T01:35:59.930 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64388.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64388.log.gz 2024-09-18T01:35:59.932 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41131.log.gz 2024-09-18T01:35:59.933 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60310.log 2024-09-18T01:35:59.934 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47841.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47841.log.gz 2024-09-18T01:35:59.934 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.37130.log 2024-09-18T01:35:59.935 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60310.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60310.log.gz 2024-09-18T01:35:59.936 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45982.log 2024-09-18T01:35:59.937 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.3.37130.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42875.log 2024-09-18T01:35:59.952 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45982.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46143.log.gz 2024-09-18T01:35:59.953 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68522.log 2024-09-18T01:35:59.956 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42875.log: gzip -5/var/log/ceph/cluster1-client.mirror.68522.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68522.log.gz 2024-09-18T01:35:59.956 INFO:teuthology.orchestra.run.smithi150.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.70800.log 2024-09-18T01:35:59.957 INFO:teuthology.orchestra.run.smithi150.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42875.log.gz 2024-09-18T01:35:59.958 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35624.log 2024-09-18T01:35:59.961 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70800.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.70800.log.gz 2024-09-18T01:35:59.965 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47105.log.gz 2024-09-18T01:35:59.973 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70445.log 2024-09-18T01:35:59.973 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.35624.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67915.log 2024-09-18T01:35:59.973 INFO:teuthology.orchestra.run.smithi150.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35624.log.gz 2024-09-18T01:35:59.973 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70445.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70445.log.gz 2024-09-18T01:35:59.975 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47083.log 2024-09-18T01:35:59.975 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67915.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67915.log.gz 2024-09-18T01:35:59.975 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57973.log 2024-09-18T01:35:59.976 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47083.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64512.log 2024-09-18T01:35:59.976 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57973.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47083.log.gz 2024-09-18T01:35:59.976 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64026.log 2024-09-18T01:35:59.977 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64512.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57973.log.gz 2024-09-18T01:35:59.977 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67802.log 2024-09-18T01:35:59.982 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64026.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64512.log.gz 2024-09-18T01:35:59.982 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64026.log.gz 2024-09-18T01:35:59.982 INFO:teuthology.orchestra.run.smithi150.stderr: -- /var/log/ceph/cluster2-mon.a.log 2024-09-18T01:35:59.982 INFO:teuthology.orchestra.run.smithi150.stderr:gzip/var/log/ceph/cluster1-client.mirror.67802.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46556.log 2024-09-18T01:35:59.982 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67802.log.gz 2024-09-18T01:35:59.982 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49049.log 2024-09-18T01:35:59.983 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54991.log 2024-09-18T01:35:59.983 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49049.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49049.log.gz 2024-09-18T01:35:59.983 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52532.log 2024-09-18T01:35:59.985 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54991.log: /var/log/ceph/cluster2-client.mirror.46556.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46556.log.gz 2024-09-18T01:35:59.985 INFO:teuthology.orchestra.run.smithi150.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54991.log.gz 2024-09-18T01:35:59.985 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44923.log 2024-09-18T01:35:59.985 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55215.log 2024-09-18T01:35:59.985 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.44923.log: /var/log/ceph/cluster2-client.mirror.52532.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44923.log.gz 2024-09-18T01:35:59.988 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55215.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55215.log.gz 2024-09-18T01:35:59.988 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52532.log.gz 2024-09-18T01:35:59.988 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39084.log 2024-09-18T01:35:59.988 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53362.log 2024-09-18T01:35:59.989 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39084.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42559.log 2024-09-18T01:35:59.989 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.39084.log.gz 2024-09-18T01:35:59.990 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53362.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60858.log 2024-09-18T01:35:59.990 INFO:teuthology.orchestra.run.smithi150.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53362.log.gz 2024-09-18T01:35:59.994 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60858.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.60858.log.gz 2024-09-18T01:35:59.994 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42559.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70206.log 2024-09-18T01:35:59.994 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45424.log 2024-09-18T01:35:59.994 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72364.log 2024-09-18T01:35:59.995 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45424.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45424.log.gz 2024-09-18T01:35:59.997 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70206.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42559.log.gz 2024-09-18T01:35:59.997 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36923.log 2024-09-18T01:35:59.997 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72364.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72364.log.gz 2024-09-18T01:35:59.998 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39105.log 2024-09-18T01:35:59.999 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36923.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36923.log.gz 2024-09-18T01:36:00.007 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39105.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65691.log 2024-09-18T01:36:00.007 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47782.log 2024-09-18T01:36:00.010 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65691.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65691.log.gz 2024-09-18T01:36:00.010 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64277.log 2024-09-18T01:36:00.010 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.37130.log.gz 2024-09-18T01:36:00.011 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47782.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47782.log.gz 2024-09-18T01:36:00.011 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70040.log 2024-09-18T01:36:00.012 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64277.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64277.log.gz 2024-09-18T01:36:00.016 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37862.log 2024-09-18T01:36:00.017 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.70040.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.70040.log.gz 2024-09-18T01:36:00.018 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49467.log 2024-09-18T01:36:00.019 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37862.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47026.log 2024-09-18T01:36:00.019 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.49467.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47635.log.gz 2024-09-18T01:36:00.020 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 2.3% --verbose -- replaced with /var/log/ceph/cluster2-client.mirror.37862.log.gz 2024-09-18T01:36:00.020 INFO:teuthology.orchestra.run.smithi150.stderr: -- /var/log/ceph/cluster1-client.mirror.43746.log 2024-09-18T01:36:00.021 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47026.log: 95.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49467.log.gz 2024-09-18T01:36:00.032 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52928.log 2024-09-18T01:36:00.039 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43746.log: 93.8% 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43746.log.gz 2024-09-18T01:36:00.039 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46657.log.gz 2024-09-18T01:36:00.039 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 93.4% --verbose -- /var/log/ceph/cluster1-client.mirror.61247.log 2024-09-18T01:36:00.039 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.39105.log.gz 2024-09-18T01:36:00.042 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56489.log 2024-09-18T01:36:00.042 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52928.log: /var/log/ceph/cluster1-client.mirror.61247.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.61247.log.gz 2024-09-18T01:36:00.042 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41769.log 2024-09-18T01:36:00.043 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.52928.log.gz 2024-09-18T01:36:00.043 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56489.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40415.log 2024-09-18T01:36:00.043 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41769.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48970.log 2024-09-18T01:36:00.043 INFO:teuthology.orchestra.run.smithi150.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.56489.log.gz 2024-09-18T01:36:00.043 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40415.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64803.log 2024-09-18T01:36:00.044 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.48970.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48970.log.gz 2024-09-18T01:36:00.047 INFO:teuthology.orchestra.run.smithi150.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41769.log.gz 2024-09-18T01:36:00.047 INFO:teuthology.orchestra.run.smithi150.stderr: -- /var/log/ceph/cluster2-client.mirror.65030.log 2024-09-18T01:36:00.047 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64803.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40415.log.gz 2024-09-18T01:36:00.047 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64803.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66250.log 2024-09-18T01:36:00.048 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:36:00.050 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65030.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65030.log.gz 2024-09-18T01:36:00.050 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.49467.log 2024-09-18T01:36:00.050 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66250.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66250.log.gz 2024-09-18T01:36:00.050 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39169.log 2024-09-18T01:36:00.052 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.1.49467.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69636.log 2024-09-18T01:36:00.055 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39169.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49672.log 2024-09-18T01:36:00.059 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.69636.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.69636.log.gz 2024-09-18T01:36:00.060 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42851.log 2024-09-18T01:36:00.060 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49672.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49672.log.gz 2024-09-18T01:36:00.061 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56995.log 2024-09-18T01:36:00.065 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42851.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54874.log 2024-09-18T01:36:00.069 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56995.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.42851.log.gz 2024-09-18T01:36:00.070 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69949.log.gz 2024-09-18T01:36:00.070 INFO:teuthology.orchestra.run.smithi150.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.56995.log.gz 2024-09-18T01:36:00.072 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52769.log 2024-09-18T01:36:00.072 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54874.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54874.log.gz 2024-09-18T01:36:00.074 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40531.log 2024-09-18T01:36:00.074 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52769.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52769.log.gz 2024-09-18T01:36:00.074 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49933.log 2024-09-18T01:36:00.074 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61199.log 2024-09-18T01:36:00.078 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49933.log: /var/log/ceph/cluster1-client.mirror.40531.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67577.log 2024-09-18T01:36:00.078 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61199.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.61199.log.gz 2024-09-18T01:36:00.079 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57203.log 2024-09-18T01:36:00.086 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67577.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.40531.log.gz 2024-09-18T01:36:00.088 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67577.log.gz 2024-09-18T01:36:00.096 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49237.log.gz 2024-09-18T01:36:00.097 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62513.log 2024-09-18T01:36:00.097 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44677.log 2024-09-18T01:36:00.098 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57203.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.57203.log.gz 2024-09-18T01:36:00.101 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42138.log 2024-09-18T01:36:00.101 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44677.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.1.49467.log.gz 2024-09-18T01:36:00.101 INFO:teuthology.orchestra.run.smithi150.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.44677.log.gz 2024-09-18T01:36:00.101 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40241.log 2024-09-18T01:36:00.101 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62513.log: /var/log/ceph/cluster1-client.admin.42138.log: gzip -5 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62513.log.gz 2024-09-18T01:36:00.101 INFO:teuthology.orchestra.run.smithi150.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.48417.log 2024-09-18T01:36:00.102 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.40241.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71664.log 2024-09-18T01:36:00.102 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48417.log: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48417.log.gz 2024-09-18T01:36:00.104 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44262.log 2024-09-18T01:36:00.104 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71664.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2024-09-18T01:36:00.104 INFO:teuthology.orchestra.run.smithi150.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.71664.log.gz 2024-09-18T01:36:00.104 INFO:teuthology.orchestra.run.smithi150.stderr: 87.5%/var/log/ceph/cluster1-client.mirror.44262.log: -- replaced with /var/log/ceph/cluster1-client.admin.40241.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53320.log 2024-09-18T01:36:00.104 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:36:00.104 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52600.log 2024-09-18T01:36:00.104 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53320.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.42138.log.gz 2024-09-18T01:36:00.107 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36671.log 2024-09-18T01:36:00.107 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52600.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52600.log.gz 2024-09-18T01:36:00.107 INFO:teuthology.orchestra.run.smithi150.stderr:gzip 92.5% -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70937.log 2024-09-18T01:36:00.107 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.53320.log.gz 2024-09-18T01:36:00.107 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.36671.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58919.log 2024-09-18T01:36:00.107 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.36671.log.gz 2024-09-18T01:36:00.107 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70937.log: gzip -5 --verbose 78.4% -- replaced with /var/log/ceph/cluster2-client.mirror.70937.log.gz 2024-09-18T01:36:00.107 INFO:teuthology.orchestra.run.smithi150.stderr: -- /var/log/ceph/cluster1-client.mirror.55167.log 2024-09-18T01:36:00.111 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58919.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58919.log.gz 2024-09-18T01:36:00.111 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69657.log 2024-09-18T01:36:00.111 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55167.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55167.log.gz 2024-09-18T01:36:00.111 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57794.log 2024-09-18T01:36:00.111 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69657.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69657.log.gz 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44262.log.gz 2024-09-18T01:36:00.111 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:36:00.111 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64865.log 2024-09-18T01:36:00.111 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57794.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36965.log 2024-09-18T01:36:00.117 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64865.log: 89.9% 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64865.log.gz 2024-09-18T01:36:00.117 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.57794.log.gz 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66565.log 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36965.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36965.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41811.log 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60446.log 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66565.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66565.log.gz 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.41811.log: -- /var/log/ceph/cluster2-client.admin.36335.log 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60446.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55705.log 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36335.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36335.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60878.log 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60446.log.gz 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr:gzip/var/log/ceph/cluster1-client.mirror.55705.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63280.log 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60878.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62804.log 2024-09-18T01:36:00.118 INFO:teuthology.orchestra.run.smithi150.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55705.log.gz 2024-09-18T01:36:00.119 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63280.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44985.log 2024-09-18T01:36:00.119 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62804.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63280.log.gz 2024-09-18T01:36:00.119 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37674.log 2024-09-18T01:36:00.119 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44985.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44985.log.gz 2024-09-18T01:36:00.119 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47993.log 2024-09-18T01:36:00.135 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.37674.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62804.log.gz 2024-09-18T01:36:00.135 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61906.log 2024-09-18T01:36:00.135 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.47993.log: -- /var/log/ceph/cluster1-client.mirror.57179.log 2024-09-18T01:36:00.135 INFO:teuthology.orchestra.run.smithi150.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47993.log.gz 2024-09-18T01:36:00.135 INFO:teuthology.orchestra.run.smithi150.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.41811.log.gz 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61906.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44597.log 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61906.log.gz 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57179.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62472.log 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44597.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57179.log.gz 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.44597.log.gz 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39554.log 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62472.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63073.log 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.62472.log.gz 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39554.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54501.log 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63073.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36797.log 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.63073.log.gz 2024-09-18T01:36:00.136 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54501.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48868.log 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36797.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36797.log.gz 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67555.log 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr: 74.0% -- replaced with /var/log/ceph/cluster1-client.mirror.39554.log.gz 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48868.log: 92.4% 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48868.log.gz 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.54501.log.gz 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr: 92.1%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68005.log 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.60878.log.gz 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67555.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67555.log.gz 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42313.log 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.68005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35997.log 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.68005.log.gz 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr:gzip/var/log/ceph/cluster2-client.mirror.42313.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63404.log 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr:gzip/var/log/ceph/cluster2-client.admin.35997.log: -5 --verbose -- 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35997.log.gz 2024-09-18T01:36:00.137 INFO:teuthology.orchestra.run.smithi150.stderr: /var/log/ceph/cluster2-client.mirror.48992.log 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63404.log: gzip 58.0% -- replaced with /var/log/ceph/cluster1-client.mirror.37674.log.gz 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr: 88.1%/var/log/ceph/cluster2-client.mirror.48992.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64471.log 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.63404.log.gz 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52689.log 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49593.log/var/log/ceph/cluster2-client.mirror.64471.log: 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52689.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64471.log.gz 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53480.log 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49593.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49593.log.gz 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70359.log 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53480.log: 87.3% 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52689.log.gz 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.53480.log.gz 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47666.log 2024-09-18T01:36:00.138 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49091.log 2024-09-18T01:36:00.139 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47666.log: /var/log/ceph/cluster1-client.mirror.70359.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47666.log.gz 2024-09-18T01:36:00.139 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49091.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49091.log.gz 2024-09-18T01:36:00.139 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.70359.log.gz 2024-09-18T01:36:00.139 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59485.log 2024-09-18T01:36:00.139 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42357.log 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59485.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59485.log.gz 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59080.log 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54788.log 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42357.log: /var/log/ceph/cluster2-client.mirror.54788.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54788.log.gz 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52347.log 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59080.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42357.log.gz 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.59080.log.gz 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45346.log 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52347.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64927.log 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45346.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52347.log.gz 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45346.log.gz 2024-09-18T01:36:00.144 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46399.log 2024-09-18T01:36:00.145 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64927.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64927.log.gz 2024-09-18T01:36:00.145 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43399.log 2024-09-18T01:36:00.145 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46399.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61971.log 2024-09-18T01:36:00.150 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43399.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.42313.log.gz 2024-09-18T01:36:00.153 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43399.log.gz 2024-09-18T01:36:00.154 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61440.log 2024-09-18T01:36:00.155 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50013.log 2024-09-18T01:36:00.155 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61440.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.61440.log.gz 2024-09-18T01:36:00.156 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65889.log 2024-09-18T01:36:00.168 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50013.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.61971.log.gz 2024-09-18T01:36:00.173 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59122.log 2024-09-18T01:36:00.174 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.65889.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65889.log.gz 2024-09-18T01:36:00.182 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46499.log.gz 2024-09-18T01:36:00.185 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50909.log 2024-09-18T01:36:00.187 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.59122.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59122.log.gz 2024-09-18T01:36:00.200 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52749.log 2024-09-18T01:36:00.201 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50909.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50909.log.gz 2024-09-18T01:36:00.201 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52048.log 2024-09-18T01:36:00.203 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52749.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52749.log.gz 2024-09-18T01:36:00.203 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63114.log 2024-09-18T01:36:00.207 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52048.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.52048.log.gz 2024-09-18T01:36:00.207 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65650.log 2024-09-18T01:36:00.213 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63114.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63114.log.gz 2024-09-18T01:36:00.216 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37691.log 2024-09-18T01:36:00.217 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65650.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65650.log.gz 2024-09-18T01:36:00.217 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66970.log 2024-09-18T01:36:00.218 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.37691.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62722.log 2024-09-18T01:36:00.222 INFO:teuthology.orchestra.run.smithi150.stderr: 53.4% -- replaced with /var/log/ceph/cluster1-client.mirror.37691.log.gz 2024-09-18T01:36:00.228 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44817.log 2024-09-18T01:36:00.232 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62722.log: 88.3%/var/log/ceph/cluster1-client.mirror.66970.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66970.log.gz 2024-09-18T01:36:00.236 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50981.log 2024-09-18T01:36:00.237 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44817.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44817.log.gz 2024-09-18T01:36:00.237 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.62722.log.gz 2024-09-18T01:36:00.237 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72490.log 2024-09-18T01:36:00.242 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50981.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50981.log.gz 2024-09-18T01:36:00.246 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69283.log 2024-09-18T01:36:00.248 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72490.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72490.log.gz 2024-09-18T01:36:00.249 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40683.log 2024-09-18T01:36:00.250 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69283.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69283.log.gz 2024-09-18T01:36:00.251 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56103.log 2024-09-18T01:36:00.251 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40683.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39795.log 2024-09-18T01:36:00.259 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56103.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56103.log.gz 2024-09-18T01:36:00.260 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44027.log 2024-09-18T01:36:00.260 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32723.log 2024-09-18T01:36:00.263 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70206.log.gz 2024-09-18T01:36:00.266 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44027.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62679.log 2024-09-18T01:36:00.267 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.32723.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32723.log.gz 2024-09-18T01:36:00.267 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71685.log 2024-09-18T01:36:00.269 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62679.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44027.log.gz 2024-09-18T01:36:00.270 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46281.log 2024-09-18T01:36:00.273 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71685.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.71685.log.gz 2024-09-18T01:36:00.273 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60612.log 2024-09-18T01:36:00.273 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62679.log.gz/var/log/ceph/cluster2-client.mirror.46281.log: 2024-09-18T01:36:00.273 INFO:teuthology.orchestra.run.smithi150.stderr:gzip 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46281.log.gz 2024-09-18T01:36:00.273 INFO:teuthology.orchestra.run.smithi150.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.37589.log 2024-09-18T01:36:00.273 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60612.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41153.log 2024-09-18T01:36:00.273 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.37589.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39465.log 2024-09-18T01:36:00.274 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41153.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60612.log.gz 2024-09-18T01:36:00.274 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39708.log 2024-09-18T01:36:00.277 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39465.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60741.log 2024-09-18T01:36:00.277 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39708.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41153.log.gz 2024-09-18T01:36:00.277 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69575.log 2024-09-18T01:36:00.278 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60741.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66475.log 2024-09-18T01:36:00.278 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60741.log.gz 2024-09-18T01:36:00.278 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69575.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48396.log 2024-09-18T01:36:00.278 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66475.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66475.log.gz 2024-09-18T01:36:00.278 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53034.log 2024-09-18T01:36:00.280 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48396.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.39465.log.gz 2024-09-18T01:36:00.285 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38863.log 2024-09-18T01:36:00.285 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53034.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53034.log.gz 2024-09-18T01:36:00.285 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66024.log 2024-09-18T01:36:00.286 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.38863.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62887.log 2024-09-18T01:36:00.293 INFO:teuthology.orchestra.run.smithi150.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.38863.log.gz 2024-09-18T01:36:00.294 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66024.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66024.log.gz 2024-09-18T01:36:00.294 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51807.log 2024-09-18T01:36:00.302 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62887.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62887.log.gz 2024-09-18T01:36:00.304 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67847.log 2024-09-18T01:36:00.304 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51807.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51807.log.gz 2024-09-18T01:36:00.311 INFO:teuthology.orchestra.run.smithi150.stderr: 66.4% -- replaced with /var/log/ceph/cluster1-client.admin.37589.log.gz 2024-09-18T01:36:00.325 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32929.log 2024-09-18T01:36:00.325 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67847.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67847.log.gz 2024-09-18T01:36:00.336 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61054.log 2024-09-18T01:36:00.336 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.32929.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.32929.log.gz 2024-09-18T01:36:00.357 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67105.log 2024-09-18T01:36:00.358 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61054.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61054.log.gz 2024-09-18T01:36:00.364 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.50138.log 2024-09-18T01:36:00.374 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67105.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65629.log 2024-09-18T01:36:00.377 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.1.50138.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67105.log.gz 2024-09-18T01:36:00.382 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72078.log 2024-09-18T01:36:00.398 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65629.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50459.log 2024-09-18T01:36:00.398 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72078.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72078.log.gz 2024-09-18T01:36:00.398 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65629.log.gz 2024-09-18T01:36:00.405 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38814.log 2024-09-18T01:36:00.415 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50459.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.40304.log 2024-09-18T01:36:00.418 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.38814.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50459.log.gz 2024-09-18T01:36:00.431 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.38814.log.gz 2024-09-18T01:36:00.436 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42606.log 2024-09-18T01:36:00.454 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.0.40304.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54272.log 2024-09-18T01:36:00.457 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.42606.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.39708.log.gz 2024-09-18T01:36:00.463 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64450.log 2024-09-18T01:36:00.468 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54272.log: 92.4% 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.42606.log.gz 2024-09-18T01:36:00.471 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.50138.log.gz 2024-09-18T01:36:00.478 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.54272.log.gz 2024-09-18T01:36:00.485 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48010.log 2024-09-18T01:36:00.487 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64450.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64450.log.gz 2024-09-18T01:36:00.500 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42494.log 2024-09-18T01:36:00.501 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48010.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48010.log.gz 2024-09-18T01:36:00.521 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44457.log 2024-09-18T01:36:00.534 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42494.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49128.log 2024-09-18T01:36:00.535 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44457.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44457.log.gz 2024-09-18T01:36:00.556 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56151.log 2024-09-18T01:36:00.556 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49128.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49128.log.gz 2024-09-18T01:36:00.566 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61175.log 2024-09-18T01:36:00.576 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56151.log: 87.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43640.log 2024-09-18T01:36:00.576 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61175.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61175.log.gz 2024-09-18T01:36:00.578 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.56151.log.gz 2024-09-18T01:36:00.582 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48992.log.gz 2024-09-18T01:36:00.583 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47593.log 2024-09-18T01:36:00.584 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43640.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49933.log.gz 2024-09-18T01:36:00.590 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43640.log.gz 2024-09-18T01:36:00.590 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46063.log 2024-09-18T01:36:00.590 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47593.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53010.log 2024-09-18T01:36:00.592 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46063.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47593.log.gz 2024-09-18T01:36:00.592 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67262.log 2024-09-18T01:36:00.593 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53010.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53010.log.gz 2024-09-18T01:36:00.593 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39513.log 2024-09-18T01:36:00.594 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67262.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67262.log.gz 2024-09-18T01:36:00.594 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35915.log 2024-09-18T01:36:00.595 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39513.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37130.log 2024-09-18T01:36:00.595 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.35915.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35915.log.gz 2024-09-18T01:36:00.596 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51491.log 2024-09-18T01:36:00.597 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39421.log 2024-09-18T01:36:00.597 INFO:teuthology.orchestra.run.smithi150.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.39513.log.gz 2024-09-18T01:36:00.599 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51491.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51491.log.gz 2024-09-18T01:36:00.599 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50756.log 2024-09-18T01:36:00.601 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39421.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56079.log 2024-09-18T01:36:00.601 INFO:teuthology.orchestra.run.smithi150.stderr: 81.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37130.log.gz 2024-09-18T01:36:00.602 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50756.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50756.log.gz 2024-09-18T01:36:00.602 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64153.log 2024-09-18T01:36:00.603 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56079.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.39421.log.gz 2024-09-18T01:36:00.603 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56079.log.gz 2024-09-18T01:36:00.604 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54183.log 2024-09-18T01:36:00.604 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64153.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64153.log.gz 2024-09-18T01:36:00.605 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70336.log 2024-09-18T01:36:00.606 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54183.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.54183.log.gz 2024-09-18T01:36:00.606 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41175.log 2024-09-18T01:36:00.608 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.70336.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.70336.log.gz 2024-09-18T01:36:00.608 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57862.log 2024-09-18T01:36:00.609 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49716.log 2024-09-18T01:36:00.609 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57862.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.57862.log.gz 2024-09-18T01:36:00.611 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48888.log 2024-09-18T01:36:00.611 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49716.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49716.log.gz 2024-09-18T01:36:00.612 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71599.log 2024-09-18T01:36:00.616 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48888.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48888.log.gz 2024-09-18T01:36:00.617 INFO:teuthology.orchestra.run.smithi150.stderr:gzip 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48396.log.gz 2024-09-18T01:36:00.619 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.71599.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.71599.log.gz 2024-09-18T01:36:00.619 INFO:teuthology.orchestra.run.smithi150.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43832.log 2024-09-18T01:36:00.619 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50865.log 2024-09-18T01:36:00.620 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.43832.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40596.log 2024-09-18T01:36:00.623 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50865.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45982.log.gz 2024-09-18T01:36:00.624 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56220.log 2024-09-18T01:36:00.624 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40596.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60938.log 2024-09-18T01:36:00.625 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56220.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43832.log.gz 2024-09-18T01:36:00.625 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42494.log.gz 2024-09-18T01:36:00.625 INFO:teuthology.orchestra.run.smithi150.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.49971.log 2024-09-18T01:36:00.628 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60938.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50865.log.gz 2024-09-18T01:36:00.629 INFO:teuthology.orchestra.run.smithi150.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56220.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36839.log 2024-09-18T01:36:00.629 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:36:00.629 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62638.log 2024-09-18T01:36:00.629 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60938.log.gz 2024-09-18T01:36:00.629 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36839.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46477.log 2024-09-18T01:36:00.629 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.36839.log.gz 2024-09-18T01:36:00.629 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51114.log 2024-09-18T01:36:00.629 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62638.log: /var/log/ceph/cluster2-client.mirror.46477.log: 87.4% 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62638.log.gz 2024-09-18T01:36:00.629 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46477.log.gz 2024-09-18T01:36:00.629 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64430.log 2024-09-18T01:36:00.632 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49971.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54898.log 2024-09-18T01:36:00.632 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:36:00.632 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.51114.log: /var/log/ceph/cluster2-client.mirror.64430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58694.log 2024-09-18T01:36:00.632 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51114.log.gz 2024-09-18T01:36:00.632 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64430.log.gz 2024-09-18T01:36:00.632 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54898.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64636.log 2024-09-18T01:36:00.632 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58694.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.54898.log.gz 2024-09-18T01:36:00.633 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69678.log 2024-09-18T01:36:00.633 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.58694.log.gz 2024-09-18T01:36:00.633 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64636.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64636.log.gz 2024-09-18T01:36:00.634 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43075.log 2024-09-18T01:36:00.636 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.69678.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69678.log.gz 2024-09-18T01:36:00.636 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43075.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67015.log 2024-09-18T01:36:00.636 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43356.log 2024-09-18T01:36:00.636 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67015.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47004.log 2024-09-18T01:36:00.636 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67015.log.gz 2024-09-18T01:36:00.637 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43356.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50483.log 2024-09-18T01:36:00.640 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48851.log 2024-09-18T01:36:00.641 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50483.log: /var/log/ceph/cluster2-client.mirror.47004.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50483.log.gz 2024-09-18T01:36:00.641 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47004.log.gz 2024-09-18T01:36:00.641 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47063.log 2024-09-18T01:36:00.641 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48851.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48851.log.gz 2024-09-18T01:36:00.641 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51403.log 2024-09-18T01:36:00.644 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47063.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57266.log 2024-09-18T01:36:00.645 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51403.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47063.log.gz 2024-09-18T01:36:00.645 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51403.log.gz 2024-09-18T01:36:00.645 INFO:teuthology.orchestra.run.smithi150.stderr: -- /var/log/ceph/cluster1-client.mirror.67780.log 2024-09-18T01:36:00.645 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.57266.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63592.log 2024-09-18T01:36:00.645 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67780.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57266.log.gz 2024-09-18T01:36:00.645 INFO:teuthology.orchestra.run.smithi150.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.67780.log.gz 2024-09-18T01:36:00.645 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60055.log 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43356.log.gz 2024-09-18T01:36:00.645 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:36:00.645 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.63592.log: -- /var/log/ceph/cluster2-client.mirror.56764.log 2024-09-18T01:36:00.645 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60055.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.60055.log.gz 2024-09-18T01:36:00.648 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60007.log 2024-09-18T01:36:00.648 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56764.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63592.log.gz 2024-09-18T01:36:00.648 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56764.log.gz 2024-09-18T01:36:00.648 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.58011.log 2024-09-18T01:36:00.648 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60007.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39205.log 2024-09-18T01:36:00.648 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.1.58011.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60007.log.gz 2024-09-18T01:36:00.648 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65800.log 2024-09-18T01:36:00.649 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.39205.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62990.log 2024-09-18T01:36:00.651 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.65800.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.65800.log.gz 2024-09-18T01:36:00.651 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45503.log 2024-09-18T01:36:00.651 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.39205.log.gz 2024-09-18T01:36:00.651 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62990.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52096.log 2024-09-18T01:36:00.651 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45503.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45503.log.gz 2024-09-18T01:36:00.653 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62990.log.gz 2024-09-18T01:36:00.653 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58987.log 2024-09-18T01:36:00.654 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52096.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52096.log.gz 2024-09-18T01:36:00.657 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69800.log 2024-09-18T01:36:00.657 INFO:teuthology.orchestra.run.smithi150.stderr:gzip/var/log/ceph/cluster1-client.mirror.58987.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50777.log 2024-09-18T01:36:00.657 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.58987.log.gz 2024-09-18T01:36:00.657 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.69800.log: 82.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69800.log.gz 2024-09-18T01:36:00.657 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58449.log 2024-09-18T01:36:00.659 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50777.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50777.log.gz 2024-09-18T01:36:00.659 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.47487.log 2024-09-18T01:36:00.659 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58449.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47487.log 2024-09-18T01:36:00.660 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.0.47487.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58449.log.gz 2024-09-18T01:36:00.663 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47487.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68644.log 2024-09-18T01:36:00.669 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47487.log.gz 2024-09-18T01:36:00.669 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52395.log 2024-09-18T01:36:00.671 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68644.log: 86.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68644.log.gz 2024-09-18T01:36:00.671 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65216.log 2024-09-18T01:36:00.672 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52395.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52395.log.gz 2024-09-18T01:36:00.672 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66046.log 2024-09-18T01:36:00.673 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65216.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65216.log.gz 2024-09-18T01:36:00.673 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.49274.log 2024-09-18T01:36:00.674 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66046.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66046.log.gz 2024-09-18T01:36:00.675 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43100.log 2024-09-18T01:36:00.684 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.0.49274.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68126.log 2024-09-18T01:36:00.690 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.43100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63032.log 2024-09-18T01:36:00.691 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68126.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.43100.log.gz 2024-09-18T01:36:00.699 INFO:teuthology.orchestra.run.smithi150.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.1.58011.log.gz 2024-09-18T01:36:00.699 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70742.log 2024-09-18T01:36:00.701 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63032.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63032.log.gz 2024-09-18T01:36:00.701 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68126.log.gz 2024-09-18T01:36:00.701 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61689.log 2024-09-18T01:36:00.703 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70742.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70742.log.gz 2024-09-18T01:36:00.703 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61271.log 2024-09-18T01:36:00.704 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61689.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.61689.log.gz 2024-09-18T01:36:00.708 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45444.log 2024-09-18T01:36:00.708 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61271.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.40596.log.gz 2024-09-18T01:36:00.708 INFO:teuthology.orchestra.run.smithi150.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.61271.log.gz 2024-09-18T01:36:00.708 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35873.log 2024-09-18T01:36:00.708 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45444.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45444.log.gz 2024-09-18T01:36:00.708 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52865.log 2024-09-18T01:36:00.709 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.35873.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.35873.log.gz 2024-09-18T01:36:00.709 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46820.log 2024-09-18T01:36:00.709 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52865.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52865.log.gz 2024-09-18T01:36:00.709 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41953.log 2024-09-18T01:36:00.711 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46820.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46820.log.gz 2024-09-18T01:36:00.711 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45960.log 2024-09-18T01:36:00.712 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41953.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46224.log 2024-09-18T01:36:00.712 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45960.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45960.log.gz 2024-09-18T01:36:00.712 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42581.log 2024-09-18T01:36:00.713 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46224.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41953.log.gz 2024-09-18T01:36:00.717 INFO:teuthology.orchestra.run.smithi150.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.47487.log.gz 2024-09-18T01:36:00.717 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61734.log 2024-09-18T01:36:00.724 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.42581.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42659.log 2024-09-18T01:36:00.728 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61734.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.47026.log.gz 2024-09-18T01:36:00.729 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46041.log 2024-09-18T01:36:00.729 INFO:teuthology.orchestra.run.smithi150.stderr: 92.5%/var/log/ceph/cluster2-client.mirror.42659.log: -- replaced with /var/log/ceph/cluster2-client.mirror.61734.log.gz 2024-09-18T01:36:00.729 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70544.log 2024-09-18T01:36:00.729 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46041.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46041.log.gz 2024-09-18T01:36:00.732 INFO:teuthology.orchestra.run.smithi150.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42659.log.gz 2024-09-18T01:36:00.732 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53766.log 2024-09-18T01:36:00.733 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70544.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65547.log 2024-09-18T01:36:00.733 INFO:teuthology.orchestra.run.smithi150.stderr: 87.6%/var/log/ceph/cluster2-client.mirror.53766.log: -- replaced with /var/log/ceph/cluster2-client.mirror.70544.log.gz 2024-09-18T01:36:00.734 INFO:teuthology.orchestra.run.smithi150.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53766.log.gz 2024-09-18T01:36:00.735 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65340.log 2024-09-18T01:36:00.735 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65547.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.65547.log.gz 2024-09-18T01:36:00.736 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53697.log 2024-09-18T01:36:00.737 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65340.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65340.log.gz 2024-09-18T01:36:00.737 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70592.log 2024-09-18T01:36:00.739 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.53697.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53697.log.gz 2024-09-18T01:36:00.739 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44737.log 2024-09-18T01:36:00.740 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.70592.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.70592.log.gz 2024-09-18T01:36:00.740 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55726.log 2024-09-18T01:36:00.742 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44737.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.44737.log.gz 2024-09-18T01:36:00.743 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67982.log 2024-09-18T01:36:00.743 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56975.log 2024-09-18T01:36:00.744 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55726.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.55726.log.gz 2024-09-18T01:36:00.744 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59730.log 2024-09-18T01:36:00.748 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56975.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56975.log.gz 2024-09-18T01:36:00.748 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71531.log 2024-09-18T01:36:00.748 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59730.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.59730.log.gz 2024-09-18T01:36:00.748 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67037.log 2024-09-18T01:36:00.750 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71531.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71531.log.gz 2024-09-18T01:36:00.751 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63902.log 2024-09-18T01:36:00.751 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67037.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67037.log.gz 2024-09-18T01:36:00.751 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41088.log 2024-09-18T01:36:00.758 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63902.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63902.log.gz 2024-09-18T01:36:00.758 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68146.log 2024-09-18T01:36:00.758 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.41088.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69907.log 2024-09-18T01:36:00.758 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68146.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68146.log.gz 2024-09-18T01:36:00.758 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62425.log 2024-09-18T01:36:00.759 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69907.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69907.log.gz 2024-09-18T01:36:00.759 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51831.log 2024-09-18T01:36:00.760 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.62425.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.62425.log.gz 2024-09-18T01:36:00.760 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55412.log 2024-09-18T01:36:00.762 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51831.log: /var/log/ceph/cluster1-client.mirror.67982.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51831.log.gz 2024-09-18T01:36:00.762 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63758.log 2024-09-18T01:36:00.765 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55412.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.55412.log.gz 2024-09-18T01:36:00.765 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50411.log 2024-09-18T01:36:00.765 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63758.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63758.log.gz 2024-09-18T01:36:00.765 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65567.log 2024-09-18T01:36:00.765 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67982.log.gz 2024-09-18T01:36:00.765 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50411.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50411.log.gz 2024-09-18T01:36:00.765 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71553.log 2024-09-18T01:36:00.767 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.65567.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.65567.log.gz 2024-09-18T01:36:00.768 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56308.log 2024-09-18T01:36:00.772 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71553.log: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.49274.log.gz 2024-09-18T01:36:00.779 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60982.log 2024-09-18T01:36:00.783 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45755.log 2024-09-18T01:36:00.783 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56308.log: /var/log/ceph/cluster1-client.mirror.60982.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.60982.log.gz 2024-09-18T01:36:00.783 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56701.log 2024-09-18T01:36:00.783 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45755.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56308.log.gz 2024-09-18T01:36:00.783 INFO:teuthology.orchestra.run.smithi150.stderr: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45755.log.gz 2024-09-18T01:36:00.783 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45466.log 2024-09-18T01:36:00.783 INFO:teuthology.orchestra.run.smithi150.stderr:gzip/var/log/ceph/cluster1-client.mirror.56701.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50435.log 2024-09-18T01:36:00.792 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45466.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50013.log.gz 2024-09-18T01:36:00.792 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71706.log 2024-09-18T01:36:00.792 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50435.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.50435.log.gz 2024-09-18T01:36:00.792 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68566.log 2024-09-18T01:36:00.792 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71706.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.56701.log.gz 2024-09-18T01:36:00.792 INFO:teuthology.orchestra.run.smithi150.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71706.log.gz 2024-09-18T01:36:00.793 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48233.log 2024-09-18T01:36:00.793 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68566.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57609.log 2024-09-18T01:36:00.793 INFO:teuthology.orchestra.run.smithi150.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68566.log.gz 2024-09-18T01:36:00.793 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61617.log 2024-09-18T01:36:00.793 INFO:teuthology.orchestra.run.smithi150.stderr: 79.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48233.log.gz 2024-09-18T01:36:00.793 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69533.log 2024-09-18T01:36:00.794 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61617.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61617.log.gz 2024-09-18T01:36:00.794 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51022.log 2024-09-18T01:36:00.794 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.69533.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69533.log.gz 2024-09-18T01:36:00.794 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62360.log 2024-09-18T01:36:00.797 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51022.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.51022.log.gz 2024-09-18T01:36:00.797 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61509.log 2024-09-18T01:36:00.797 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.62360.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.62360.log.gz 2024-09-18T01:36:00.797 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56677.log 2024-09-18T01:36:00.797 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57609.log: /var/log/ceph/cluster2-client.mirror.61509.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.57609.log.gz 2024-09-18T01:36:00.798 INFO:teuthology.orchestra.run.smithi150.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61509.log.gz 2024-09-18T01:36:00.800 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56561.log 2024-09-18T01:36:00.800 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56677.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56677.log.gz 2024-09-18T01:36:00.800 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49652.log 2024-09-18T01:36:00.800 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37930.log 2024-09-18T01:36:00.802 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49652.log: /var/log/ceph/cluster2-client.mirror.56561.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56561.log.gz 2024-09-18T01:36:00.802 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54605.log 2024-09-18T01:36:00.803 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37930.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36209.log 2024-09-18T01:36:00.803 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54605.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54605.log.gz 2024-09-18T01:36:00.809 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59284.log 2024-09-18T01:36:00.809 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36209.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36209.log.gz 2024-09-18T01:36:00.809 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45873.log 2024-09-18T01:36:00.809 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59284.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59284.log.gz 2024-09-18T01:36:00.809 INFO:teuthology.orchestra.run.smithi150.stderr: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.37930.log.gz 2024-09-18T01:36:00.809 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55854.log 2024-09-18T01:36:00.809 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45873.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.45873.log.gz 2024-09-18T01:36:00.809 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49012.log 2024-09-18T01:36:00.809 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55854.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.55854.log.gz 2024-09-18T01:36:00.810 INFO:teuthology.orchestra.run.smithi150.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49652.log.gz 2024-09-18T01:36:00.810 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61814.log 2024-09-18T01:36:00.810 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59212.log 2024-09-18T01:36:00.810 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49012.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49012.log.gz 2024-09-18T01:36:00.811 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55320.log 2024-09-18T01:36:00.811 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59212.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.59212.log.gz 2024-09-18T01:36:00.811 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54456.log 2024-09-18T01:36:00.814 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55320.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55320.log.gz 2024-09-18T01:36:00.814 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64823.log 2024-09-18T01:36:00.814 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.61814.log: /var/log/ceph/cluster1-client.mirror.54456.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.54456.log.gz 2024-09-18T01:36:00.814 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58220.log 2024-09-18T01:36:00.817 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64823.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64823.log.gz 2024-09-18T01:36:00.817 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67397.log 2024-09-18T01:36:00.819 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58220.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.58220.log.gz 2024-09-18T01:36:00.819 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40078.log 2024-09-18T01:36:00.819 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67397.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67397.log.gz 2024-09-18T01:36:00.820 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46598.log 2024-09-18T01:36:00.820 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40078.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37811.log 2024-09-18T01:36:00.821 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46598.log: 79.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46598.log.gz 2024-09-18T01:36:00.822 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.61814.log.gz 2024-09-18T01:36:00.822 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63052.log 2024-09-18T01:36:00.823 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.37811.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59236.log 2024-09-18T01:36:00.824 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63052.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41088.log.gz 2024-09-18T01:36:00.826 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66812.log 2024-09-18T01:36:00.826 INFO:teuthology.orchestra.run.smithi150.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63052.log.gz 2024-09-18T01:36:00.827 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59236.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43483.log 2024-09-18T01:36:00.827 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66812.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66812.log.gz 2024-09-18T01:36:00.827 INFO:teuthology.orchestra.run.smithi150.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59236.log.gz 2024-09-18T01:36:00.830 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.43483.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.43483.log.gz 2024-09-18T01:36:00.830 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- 66.4% -- replaced with /var/log/ceph/cluster2-client.admin.37811.log.gz 2024-09-18T01:36:00.831 INFO:teuthology.orchestra.run.smithi150.stderr: /var/log/ceph/cluster2-client.mirror.47861.log 2024-09-18T01:36:00.831 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49274.log 2024-09-18T01:36:00.831 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47861.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64782.log 2024-09-18T01:36:00.833 INFO:teuthology.orchestra.run.smithi150.stderr: 87.5%/var/log/ceph/cluster2-client.mirror.49274.log: -- replaced with /var/log/ceph/cluster2-client.mirror.47861.log.gz 2024-09-18T01:36:00.834 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48772.log 2024-09-18T01:36:00.835 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64782.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64782.log.gz 2024-09-18T01:36:00.835 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71620.log 2024-09-18T01:36:00.839 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48772.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48772.log.gz 2024-09-18T01:36:00.842 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54360.log 2024-09-18T01:36:00.844 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71620.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71620.log.gz 2024-09-18T01:36:00.844 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.log 2024-09-18T01:36:00.848 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54360.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54360.log.gz 2024-09-18T01:36:00.852 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42696.log 2024-09-18T01:36:00.858 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2.log: 87.4% -- replaced with /var/log/ceph/cluster2.log.gz 2024-09-18T01:36:00.858 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50819.log 2024-09-18T01:36:00.859 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42696.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50885.log 2024-09-18T01:36:00.861 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50819.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50819.log.gz 2024-09-18T01:36:00.861 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62970.log 2024-09-18T01:36:00.863 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50885.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50885.log.gz 2024-09-18T01:36:00.863 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45064.log 2024-09-18T01:36:00.863 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62970.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.62970.log.gz 2024-09-18T01:36:00.864 INFO:teuthology.orchestra.run.smithi150.stderr: 82.7% -- replaced with /var/log/ceph/cluster1-client.mirror.42696.log.gz 2024-09-18T01:36:00.865 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37657.log 2024-09-18T01:36:00.870 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45064.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.45064.log.gz 2024-09-18T01:36:00.870 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46399.log.gz 2024-09-18T01:36:00.874 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.37657.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66902.log 2024-09-18T01:36:00.874 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50797.log 2024-09-18T01:36:00.875 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66902.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66902.log.gz 2024-09-18T01:36:00.875 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49991.log 2024-09-18T01:36:00.875 INFO:teuthology.orchestra.run.smithi150.stderr: 56.7% -- replaced with /var/log/ceph/cluster1-client.mirror.37657.log.gz 2024-09-18T01:36:00.875 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.50797.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40078.log.gz 2024-09-18T01:36:00.881 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64215.log 2024-09-18T01:36:00.881 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49991.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49991.log.gz 2024-09-18T01:36:00.881 INFO:teuthology.orchestra.run.smithi150.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.50797.log.gz 2024-09-18T01:36:00.881 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.37285.log 2024-09-18T01:36:00.881 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64215.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63197.log 2024-09-18T01:36:00.881 INFO:teuthology.orchestra.run.smithi150.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64215.log.gz 2024-09-18T01:36:00.881 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.3.37285.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67082.log 2024-09-18T01:36:00.881 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.63197.log: /var/log/ceph/cluster2-client.mirror.49854.log 2024-09-18T01:36:00.881 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67082.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63197.log.gz 2024-09-18T01:36:00.881 INFO:teuthology.orchestra.run.smithi150.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67082.log.gz 2024-09-18T01:36:00.882 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50636.log 2024-09-18T01:36:00.882 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49854.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64844.log 2024-09-18T01:36:00.885 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.50636.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50636.log.gz 2024-09-18T01:36:00.896 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59100.log 2024-09-18T01:36:00.896 INFO:teuthology.orchestra.run.smithi150.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49274.log.gz 2024-09-18T01:36:00.898 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64844.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64844.log.gz 2024-09-18T01:36:00.900 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44477.log 2024-09-18T01:36:00.900 INFO:teuthology.orchestra.run.smithi150.stderr:gzip 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59100.log.gz 2024-09-18T01:36:00.902 INFO:teuthology.orchestra.run.smithi150.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58718.log 2024-09-18T01:36:00.906 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51427.log 2024-09-18T01:36:00.906 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44477.log: /var/log/ceph/cluster1-client.mirror.58718.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44477.log.gz 2024-09-18T01:36:00.906 INFO:teuthology.orchestra.run.smithi150.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.58718.log.gz 2024-09-18T01:36:00.906 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47576.log 2024-09-18T01:36:00.906 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51427.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70763.log 2024-09-18T01:36:00.906 INFO:teuthology.orchestra.run.smithi150.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51427.log.gz 2024-09-18T01:36:00.907 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70763.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70763.log.gz 2024-09-18T01:36:00.910 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47576.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47576.log.gz 2024-09-18T01:36:00.910 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48514.log 2024-09-18T01:36:00.910 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63840.log 2024-09-18T01:36:00.911 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.48514.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55609.log 2024-09-18T01:36:00.911 INFO:teuthology.orchestra.run.smithi150.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48514.log.gz 2024-09-18T01:36:00.911 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.63840.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46377.log 2024-09-18T01:36:00.911 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.55609.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.55609.log.gz 2024-09-18T01:36:00.912 INFO:teuthology.orchestra.run.smithi150.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63840.log.gz 2024-09-18T01:36:00.913 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46377.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45007.log 2024-09-18T01:36:00.914 INFO:teuthology.orchestra.run.smithi150.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46377.log.gz 2024-09-18T01:36:00.914 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51691.log 2024-09-18T01:36:00.914 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52886.log 2024-09-18T01:36:00.915 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45007.log: /var/log/ceph/cluster1-client.mirror.51691.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.51691.log.gz 2024-09-18T01:36:00.917 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52648.log 2024-09-18T01:36:00.927 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52886.log: /var/log/ceph/cluster1-client.mirror.52648.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67127.log 2024-09-18T01:36:00.928 INFO:teuthology.orchestra.run.smithi150.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52886.log.gz 2024-09-18T01:36:00.935 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52648.log.gz 2024-09-18T01:36:00.935 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38792.log 2024-09-18T01:36:00.939 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67127.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67127.log.gz 2024-09-18T01:36:00.942 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60165.log 2024-09-18T01:36:00.943 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.38792.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54027.log 2024-09-18T01:36:00.945 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.60165.log: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.60165.log.gz 2024-09-18T01:36:00.945 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62617.log 2024-09-18T01:36:00.952 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.38792.log.gz 2024-09-18T01:36:00.954 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54027.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54027.log.gz 2024-09-18T01:36:00.956 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46224.log.gz 2024-09-18T01:36:00.958 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62617.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67690.log 2024-09-18T01:36:00.958 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62617.log.gz 2024-09-18T01:36:00.960 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67690.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67690.log.gz 2024-09-18T01:36:00.961 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60633.log 2024-09-18T01:36:00.961 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42429.log 2024-09-18T01:36:00.970 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60633.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60633.log.gz 2024-09-18T01:36:00.970 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37640.log 2024-09-18T01:36:00.970 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42429.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51355.log 2024-09-18T01:36:00.970 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.37640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46578.log 2024-09-18T01:36:00.970 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.51355.log: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37640.log.gz 2024-09-18T01:36:00.970 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51355.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53649.log 2024-09-18T01:36:00.970 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:36:00.970 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37623.log 2024-09-18T01:36:00.970 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53649.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53649.log.gz 2024-09-18T01:36:00.970 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53528.log 2024-09-18T01:36:00.971 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.37623.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39125.log 2024-09-18T01:36:00.971 INFO:teuthology.orchestra.run.smithi150.stderr: 2.3% -- replaced with /var/log/ceph/cluster1-client.mirror.37623.log.gz 2024-09-18T01:36:00.971 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46578.log: /var/log/ceph/cluster1-client.mirror.53528.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53528.log.gz 2024-09-18T01:36:00.971 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64236.log 2024-09-18T01:36:00.973 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39125.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52117.log 2024-09-18T01:36:00.974 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.64236.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.64236.log.gz 2024-09-18T01:36:00.975 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70657.log 2024-09-18T01:36:00.977 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52117.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52117.log.gz 2024-09-18T01:36:00.977 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39985.log 2024-09-18T01:36:00.978 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.70657.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.70657.log.gz 2024-09-18T01:36:00.978 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40214.log 2024-09-18T01:36:00.979 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39985.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72280.log 2024-09-18T01:36:00.983 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.40214.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.39985.log.gz 2024-09-18T01:36:00.983 INFO:teuthology.orchestra.run.smithi150.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.40214.log.gz 2024-09-18T01:36:01.004 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47762.log 2024-09-18T01:36:01.004 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.72280.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.72280.log.gz 2024-09-18T01:36:01.005 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58399.log 2024-09-18T01:36:01.009 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47762.log: 89.6% 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47762.log.gz 2024-09-18T01:36:01.009 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.0.40304.log.gz 2024-09-18T01:36:01.009 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46783.log 2024-09-18T01:36:01.012 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.58399.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.58399.log.gz 2024-09-18T01:36:01.012 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37896.log 2024-09-18T01:36:01.012 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46783.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46783.log.gz 2024-09-18T01:36:01.013 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41646.log 2024-09-18T01:36:01.013 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37896.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37099.log 2024-09-18T01:36:01.014 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41646.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62288.log 2024-09-18T01:36:01.015 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37099.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49195.log 2024-09-18T01:36:01.017 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41646.log.gz 2024-09-18T01:36:01.017 INFO:teuthology.orchestra.run.smithi150.stderr: 57.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37896.log.gz 2024-09-18T01:36:01.020 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.62288.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.62288.log.gz 2024-09-18T01:36:01.020 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70125.log 2024-09-18T01:36:01.022 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49195.log: 81.7% 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49195.log.gz 2024-09-18T01:36:01.022 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.37099.log.gz 2024-09-18T01:36:01.022 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57910.log 2024-09-18T01:36:01.028 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70125.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70125.log.gz 2024-09-18T01:36:01.029 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.46694.log 2024-09-18T01:36:01.035 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57910.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.57910.log.gz 2024-09-18T01:36:01.036 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39822.log 2024-09-18T01:36:01.037 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.46694.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.46694.log.gz 2024-09-18T01:36:01.038 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42381.log 2024-09-18T01:36:01.038 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.admin.39822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56262.log 2024-09-18T01:36:01.039 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.42381.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54159.log 2024-09-18T01:36:01.040 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56262.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.39822.log.gz 2024-09-18T01:36:01.041 INFO:teuthology.orchestra.run.smithi150.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56262.log.gz 2024-09-18T01:36:01.043 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69575.log.gz 2024-09-18T01:36:01.045 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.54159.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42381.log.gz 2024-09-18T01:36:01.045 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39489.log 2024-09-18T01:36:01.046 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.54159.log.gz 2024-09-18T01:36:01.046 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47824.log 2024-09-18T01:36:01.046 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61665.log 2024-09-18T01:36:01.046 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47824.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47824.log.gz 2024-09-18T01:36:01.050 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49541.log 2024-09-18T01:36:01.051 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61665.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61665.log.gz 2024-09-18T01:36:01.051 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49795.log 2024-09-18T01:36:01.051 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49541.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49541.log.gz 2024-09-18T01:36:01.051 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57432.log 2024-09-18T01:36:01.051 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49795.log: 79.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49795.log.gz 2024-09-18T01:36:01.051 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65821.log 2024-09-18T01:36:01.078 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.57432.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.57432.log.gz 2024-09-18T01:36:01.078 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67510.log 2024-09-18T01:36:01.078 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.65821.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65821.log.gz 2024-09-18T01:36:01.078 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48497.log 2024-09-18T01:36:01.079 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.67510.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67510.log.gz 2024-09-18T01:36:01.079 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61464.log 2024-09-18T01:36:01.079 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.39489.log: /var/log/ceph/cluster2-client.mirror.48497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56882.log 2024-09-18T01:36:01.079 INFO:teuthology.orchestra.run.smithi150.stderr: 79.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48497.log.gz 2024-09-18T01:36:01.079 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61464.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37708.log 2024-09-18T01:36:01.079 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.56882.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56882.log.gz 2024-09-18T01:36:01.080 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46536.log 2024-09-18T01:36:01.080 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.37708.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62131.log 2024-09-18T01:36:01.080 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46536.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.39489.log.gz 2024-09-18T01:36:01.080 INFO:teuthology.orchestra.run.smithi150.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46536.log.gz 2024-09-18T01:36:01.080 INFO:teuthology.orchestra.run.smithi150.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.61464.log.gz 2024-09-18T01:36:01.080 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60240.log 2024-09-18T01:36:01.081 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.62131.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.62131.log.gz 2024-09-18T01:36:01.081 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45940.log 2024-09-18T01:36:01.081 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.60240.log: 54.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37708.log.gz 2024-09-18T01:36:01.081 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61930.log 2024-09-18T01:36:01.081 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45940.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60240.log.gz 2024-09-18T01:36:01.082 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49615.log 2024-09-18T01:36:01.082 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.61930.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45940.log.gz 2024-09-18T01:36:01.082 INFO:teuthology.orchestra.run.smithi150.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61930.log.gz 2024-09-18T01:36:01.082 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54340.log 2024-09-18T01:36:01.082 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.49615.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39258.log 2024-09-18T01:36:01.083 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.54340.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54340.log.gz 2024-09-18T01:36:01.083 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70380.log 2024-09-18T01:36:01.083 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.39258.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.42696.log 2024-09-18T01:36:01.083 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70380.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70380.log.gz 2024-09-18T01:36:01.083 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.27851.log 2024-09-18T01:36:01.083 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.0.42696.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70466.log 2024-09-18T01:36:01.084 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/ceph.tmp-client.admin.27851.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.27851.log.gz 2024-09-18T01:36:01.092 INFO:teuthology.orchestra.run.smithi150.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.0.42696.log.gz 2024-09-18T01:36:01.092 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40619.log 2024-09-18T01:36:01.092 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.70466.log: -- replaced with /var/log/ceph/cluster1-client.mirror.39125.log.gz 2024-09-18T01:36:01.093 INFO:teuthology.orchestra.run.smithi150.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70466.log.gz 2024-09-18T01:36:01.093 INFO:teuthology.orchestra.run.smithi150.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42429.log.gz 2024-09-18T01:36:01.093 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55585.log 2024-09-18T01:36:01.093 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.40619.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47450.log 2024-09-18T01:36:01.093 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.55585.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55585.log.gz 2024-09-18T01:36:01.093 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44837.log 2024-09-18T01:36:01.093 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.47450.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39258.log.gz 2024-09-18T01:36:01.099 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46944.log 2024-09-18T01:36:01.099 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44837.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.44837.log.gz 2024-09-18T01:36:01.110 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46063.log.gz 2024-09-18T01:36:01.111 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37178.log 2024-09-18T01:36:01.124 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46944.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71112.log 2024-09-18T01:36:01.127 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.37178.log: 85.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37178.log.gz 2024-09-18T01:36:01.137 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36461.log 2024-09-18T01:36:01.139 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.71112.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71112.log.gz 2024-09-18T01:36:01.154 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66655.log 2024-09-18T01:36:01.154 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.admin.36461.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.36461.log.gz 2024-09-18T01:36:01.167 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52137.log 2024-09-18T01:36:01.167 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.66655.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.66655.log.gz 2024-09-18T01:36:01.181 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68109.log 2024-09-18T01:36:01.182 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.52137.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.52137.log.gz 2024-09-18T01:36:01.203 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45466.log.gz 2024-09-18T01:36:01.203 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52323.log 2024-09-18T01:36:01.203 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.68109.log: 79.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68109.log.gz 2024-09-18T01:36:01.204 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53203.log 2024-09-18T01:36:01.204 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.52323.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52323.log.gz 2024-09-18T01:36:01.204 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44948.log 2024-09-18T01:36:01.204 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.53203.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53203.log.gz 2024-09-18T01:36:01.204 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40033.log 2024-09-18T01:36:01.205 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.44948.log: 79.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44948.log.gz 2024-09-18T01:36:01.205 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46357.log 2024-09-18T01:36:01.205 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.40033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45283.log 2024-09-18T01:36:01.205 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.46357.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46357.log.gz 2024-09-18T01:36:01.205 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 88.1% --verbose -- /var/log/ceph/cluster2-client.mirror.37022.log 2024-09-18T01:36:01.205 INFO:teuthology.orchestra.run.smithi150.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.40033.log.gz 2024-09-18T01:36:01.206 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.45283.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40619.log.gz 2024-09-18T01:36:01.207 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56013.log 2024-09-18T01:36:01.217 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.37022.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59778.log 2024-09-18T01:36:01.218 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.56013.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.56013.log.gz 2024-09-18T01:36:01.219 INFO:teuthology.orchestra.run.smithi150.stderr: 81.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37022.log.gz 2024-09-18T01:36:01.231 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41668.log 2024-09-18T01:36:01.232 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster1-client.mirror.59778.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59778.log.gz 2024-09-18T01:36:01.303 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/cluster2-client.mirror.41668.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46578.log.gz 2024-09-18T01:36:01.307 INFO:teuthology.orchestra.run.smithi150.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.37285.log.gz 2024-09-18T01:36:01.328 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45007.log.gz 2024-09-18T01:36:01.380 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47450.log.gz 2024-09-18T01:36:01.407 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71553.log.gz 2024-09-18T01:36:01.418 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49854.log.gz 2024-09-18T01:36:01.448 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.46944.log.gz 2024-09-18T01:36:01.511 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49615.log.gz 2024-09-18T01:36:01.666 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45283.log.gz 2024-09-18T01:36:01.709 INFO:teuthology.orchestra.run.smithi150.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.37059.log.gz 2024-09-18T01:36:04.141 INFO:teuthology.orchestra.run.smithi078.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2024-09-18T01:36:04.724 INFO:teuthology.orchestra.run.smithi078.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2024-09-18T01:36:05.601 INFO:teuthology.orchestra.run.smithi150.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2024-09-18T01:36:05.753 INFO:teuthology.orchestra.run.smithi078.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2024-09-18T01:36:05.755 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-18T01:36:05.755 INFO:teuthology.orchestra.run.smithi078.stderr:real 0m8.930s 2024-09-18T01:36:05.755 INFO:teuthology.orchestra.run.smithi078.stderr:user 0m24.681s 2024-09-18T01:36:05.755 INFO:teuthology.orchestra.run.smithi078.stderr:sys 0m0.926s 2024-09-18T01:36:06.695 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41175.log.gz 2024-09-18T01:36:07.169 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43075.log.gz 2024-09-18T01:36:07.493 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.39795.log.gz 2024-09-18T01:36:07.499 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40683.log.gz 2024-09-18T01:36:07.664 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42081.log.gz 2024-09-18T01:36:07.874 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39169.log.gz 2024-09-18T01:36:10.073 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41668.log.gz 2024-09-18T01:36:12.863 INFO:teuthology.orchestra.run.smithi150.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40185.log.gz 2024-09-18T01:36:12.887 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42581.log.gz 2024-09-18T01:38:11.583 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2024-09-18T01:38:16.883 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2024-09-18T01:38:40.714 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2024-09-18T01:38:40.716 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-09-18T01:38:40.716 INFO:teuthology.orchestra.run.smithi150.stderr:real 2m43.858s 2024-09-18T01:38:40.716 INFO:teuthology.orchestra.run.smithi150.stderr:user 4m12.377s 2024-09-18T01:38:40.716 INFO:teuthology.orchestra.run.smithi150.stderr:sys 0m11.402s 2024-09-18T01:38:40.716 INFO:tasks.ceph:Archiving logs... 2024-09-18T01:38:40.717 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686/remote/smithi078/log 2024-09-18T01:38:40.718 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-18T01:38:44.149 DEBUG:teuthology.misc:Transferring archived files from smithi150:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686/remote/smithi150/log 2024-09-18T01:38:44.150 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-18T01:39:21.854 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-18T01:39:21.869 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-18T01:39:21.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-18T01:39:21.873 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-18T01:39:21.953 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-18T01:39:21.964 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-18T01:39:21.964 DEBUG:teuthology.orchestra.run.smithi078:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-18T01:39:21.967 DEBUG:teuthology.orchestra.run.smithi150:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-18T01:39:21.982 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-09-18T01:39:21.984 INFO:teuthology.orchestra.run.smithi150.stderr:bash: line 1: ntpq: command not found 2024-09-18T01:39:21.987 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-18T01:39:21.987 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-09-18T01:39:21.987 INFO:teuthology.orchestra.run.smithi078.stdout:^* hv01.front.sepia.ceph.com 3 6 377 37 -212us[ -206us] +/- 59ms 2024-09-18T01:39:21.987 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 38 -1309us[-1303us] +/- 52ms 2024-09-18T01:39:21.987 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv03.front.sepia.ceph.com 3 7 377 37 +2192us[+2192us] +/- 83ms 2024-09-18T01:39:21.987 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T01:39:22.078 INFO:teuthology.orchestra.run.smithi150.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-18T01:39:22.078 INFO:teuthology.orchestra.run.smithi150.stdout:=============================================================================== 2024-09-18T01:39:22.078 INFO:teuthology.orchestra.run.smithi150.stdout:^* hv01.front.sepia.ceph.com 3 6 377 39 -171us[ -158us] +/- 59ms 2024-09-18T01:39:22.078 INFO:teuthology.orchestra.run.smithi150.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 39 -1281us[-1281us] +/- 52ms 2024-09-18T01:39:22.078 INFO:teuthology.orchestra.run.smithi150.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 41 +2194us[+2206us] +/- 83ms 2024-09-18T01:39:22.078 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-18T01:39:22.079 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-18T01:39:22.091 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-18T01:39:22.091 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-18T01:39:22.163 DEBUG:teuthology.orchestra.run.smithi078:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-18T01:39:22.236 DEBUG:teuthology.orchestra.run.smithi150:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-18T01:39:22.345 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-18T01:39:22.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T01:39:22.373 DEBUG:teuthology.orchestra.run.smithi150:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-18T01:39:22.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T01:39:22.404 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-18T01:39:22.418 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-18T01:39:22.438 INFO:teuthology.task.internal:Duration was 2922.227937 seconds 2024-09-18T01:39:22.439 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-18T01:39:22.470 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-18T01:39:22.471 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-18T01:39:22.473 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-18T01:39:22.510 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-18T01:39:22.540 INFO:teuthology.orchestra.run.smithi150.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-18T01:39:22.974 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-18T01:39:22.974 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi078.front.sepia.ceph.com 2024-09-18T01:39:22.974 DEBUG:teuthology.orchestra.run.smithi078:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-18T01:39:23.019 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi150.front.sepia.ceph.com 2024-09-18T01:39:23.020 DEBUG:teuthology.orchestra.run.smithi150:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-18T01:39:23.043 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-18T01:39:23.044 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-18T01:39:23.062 DEBUG:teuthology.orchestra.run.smithi150:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-18T01:39:23.113 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-18T01:39:23.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-18T01:39:23.130 DEBUG:teuthology.orchestra.run.smithi150:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-18T01:39:23.266 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-18T01:39:23.280 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-18T01:39:23.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-18T01:39:23.305 DEBUG:teuthology.orchestra.run.smithi150:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-18T01:39:23.344 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-18T01:39:23.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-18T01:39:23.359 DEBUG:teuthology.orchestra.run.smithi150:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-18T01:39:23.381 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = core 2024-09-18T01:39:23.422 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern = core 2024-09-18T01:39:23.441 DEBUG:teuthology.orchestra.run.smithi078:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-18T01:39:23.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T01:39:23.486 DEBUG:teuthology.orchestra.run.smithi150:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-18T01:39:23.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-18T01:39:23.501 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-18T01:39:23.514 INFO:teuthology.task.internal:Transferring archived files... 2024-09-18T01:39:23.514 DEBUG:teuthology.misc:Transferring archived files from smithi078:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686/remote/smithi078 2024-09-18T01:39:23.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-18T01:39:23.560 DEBUG:teuthology.misc:Transferring archived files from smithi150:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-17_21:16:02-rbd-squid-distro-default-smithi/7909686/remote/smithi150 2024-09-18T01:39:23.561 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-18T01:39:23.594 INFO:teuthology.task.internal:Removing archive directory... 2024-09-18T01:39:23.594 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-18T01:39:23.597 DEBUG:teuthology.orchestra.run.smithi150:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-18T01:39:23.644 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-18T01:39:23.658 INFO:teuthology.task.internal:Not uploading archives. 2024-09-18T01:39:23.658 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-18T01:39:23.731 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-18T01:39:23.731 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-18T01:39:23.734 DEBUG:teuthology.orchestra.run.smithi150:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-18T01:39:23.746 INFO:teuthology.orchestra.run.smithi078.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 18 01:39 /home/ubuntu/cephtest 2024-09-18T01:39:23.748 INFO:teuthology.orchestra.run.smithi150.stdout: 265336 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 18 01:39 /home/ubuntu/cephtest 2024-09-18T01:39:23.749 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-18T01:39:23.762 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-18T01:39:23.833 INFO:teuthology.run:Summary data: description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-lz4 policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock} duration: 2922.227937221527 flavor: default owner: scheduled_teuthology@teuthology success: true 2024-09-18T01:39:23.833 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-18T01:39:23.917 INFO:teuthology.run:pass